00000080:00000010:1.1:1761230875.293928:0:91021:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004d107b75. 00010000:00000010:1.1:1761230875.459054:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761230875.459069:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c9e10b74. 00000080:00000010:1.1:1761230875.579719:0:90988:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b5773b23. 00010000:00000010:1.1:1761230875.966890:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001263d429. 00010000:00000010:1.1:1761230875.966904:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00000080:00000010:1.1:1761230875.994945:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b15036c3. 00010000:00000010:1.1:1761230876.062238:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d067cc47. 00010000:00000010:1.1:1761230876.062257:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:1.1:1761230876.173879:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c540c55. 00000080:00000010:1.1:1761230876.269954:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fd1cf851. 00000080:00000010:1.1:1761230876.399002:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003ab7f06a. 00000080:00000010:1.1:1761230876.759735:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f05b7367. 00010000:00000010:1.1:1761230876.829482:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:1.1:1761230876.829499:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00000080:00000010:1.1:1761230876.883840:0:90988:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b15036c3. 00010000:00000010:1.1:1761230876.928871:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000072ba10ef. 00010000:00000010:1.1:1761230876.928888:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00010000:00000010:1.1:1761230877.075946:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007737bfd7. 00010000:00000010:1.1:1761230877.075962:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00000080:00000010:1.1:1761230877.190189:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000930c75c8. 00000080:00000010:1.1:1761230877.458079:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000057ae30e1. 00010000:00000010:1.1:1761230877.485089:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000024e22a7. 00010000:00000010:1.1:1761230877.485099:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00010000:00000010:1.1:1761230877.670079:0:91070:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b16bfd7c. 00010000:00000010:1.1:1761230877.670094:0:91070:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00010000:00000010:1.1:1761230877.726995:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761230877.727009:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00000080:00000010:1.1:1761230877.748765:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d1785496. 00010000:00000010:1.1:1761230877.784819:0:91071:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000056bceda5. 00010000:00000010:1.1:1761230877.784834:0:91071:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00000080:00000010:1.1:1761230877.795895:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000027a3870d. 00010000:00000010:1.1:1761230877.864360:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761230877.864369:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006f6778c1. 00000080:00000010:1.1:1761230877.891424:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003987a3f8. 00010000:00000010:1.1:1761230877.949961:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c682eeae. 00010000:00000010:1.1:1761230877.949977:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:1.1:1761230877.978272:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003119fc9c. 00010000:00000010:1.1:1761230878.091136:0:91076:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761230878.091155:0:91076:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:1.1:1761230878.167473:0:91077:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004620259c. 00010000:00000010:1.1:1761230878.314271:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000084a22e8f. 00010000:00000010:1.1:1761230878.314284:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:1.1:1761230878.324875:0:90988:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000bbabe4bb. 00000080:00000010:1.1:1761230878.366872:0:91081:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c240b4ed. 00010000:00000010:1.1:1761230878.405268:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000084a22e8f. 00010000:00000010:1.1:1761230878.405286:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:1.1:1761230878.422887:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003e1f952c. 00010000:00000010:1.1:1761230878.468005:0:91083:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761230878.468017:0:91083:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:1.1:1761230878.482085:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000021aeeb0a. 00010000:00000010:1.1:1761230878.693435:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000084a22e8f. 00010000:00000010:1.1:1761230878.693454:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:1.1:1761230878.714883:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000051c49ffd. 00000080:00000010:1.1:1761230878.764408:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008095118d. 00010000:00000010:1.1:1761230878.862088:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f37cde4b. 00010000:00000010:1.1:1761230878.862106:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:1.1:1761230878.880877:0:91091:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000026945b88. 00000080:00000010:1.1:1761230878.997950:0:90988:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003788de3a. 00010000:00000010:1.1:1761230879.322902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:1.1:1761230879.322914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00010000:00000010:1.1:1761230879.401319:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:1.1:1761230879.401329:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00000080:00000010:1.1:1761230879.722892:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a35479a0. 00000080:00000010:1.1:1761230879.934990:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b2cb5a1d. 00000080:00000010:1.1:1761230880.372904:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000647ca513. 00010000:00000010:1.1:1761230880.418886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:1.1:1761230880.418915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c83761be. 00010000:00000010:1.1:1761230880.690908:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:1.1:1761230880.690920:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00010000:00000010:1.1:1761230880.848925:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:1.1:1761230880.848937:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c9e10b74. 00000080:00000010:1.1:1761230880.859861:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e9822c4e. 00010000:00000010:1.1:1761230880.881909:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:1.1:1761230880.881918:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ff1e5574. 00000080:00000010:1.1:1761230880.918901:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003278ff9d. 00000080:00000010:1.1:1761230881.009770:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00000080:00000010:1.1:1761230881.165913:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e95fac7. 00000080:00000010:1.1:1761230881.431114:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000897c7016. 00000080:00000010:1.1:1761230881.464890:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000910e0743. 00000080:00000010:1.1:1761230881.580071:0:91150:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000ef54fa9. 00000080:00000010:1.1:1761230881.615718:0:91151:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000910e0743. 00010000:00000010:1.1:1761230881.665289:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:1.1:1761230881.665299:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00010000:00000010:1.1:1761230881.692925:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:1.1:1761230881.692942:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00010000:00000010:1.1:1761230881.746419:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:1.1:1761230881.746427:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c83761be. 00010000:00000010:1.1:1761230881.786112:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:1.1:1761230881.786125:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00010000:00000010:1.1:1761230881.830308:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:1.1:1761230881.830318:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:1.1:1761230881.875912:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003ace7c30. 00000080:00000010:1.1:1761230882.043183:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ff74b385. 00010000:00000010:1.1:1761230882.100016:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761230882.100032:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:1.1:1761230882.117901:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000002f170e5d. 00010000:00000010:3.1:1761230882.190901:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:3.1:1761230882.190935:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:3.1:1761230882.233168:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:3.1:1761230882.233185:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00010000:00000010:3.1:1761230882.283920:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:3.1:1761230882.283937:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00010000:00000010:1.1:1761230882.325896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c682eeae. 00010000:00000010:1.1:1761230882.325910:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003b5da300. 00010000:00000010:1.1:1761230882.355021:0:91167:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:1.1:1761230882.355036:0:91167:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00010000:00000010:1.1:1761230882.527077:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:1.1:1761230882.527085:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005ee4f696. 00010000:00000010:3.1:1761230882.605329:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:3.1:1761230882.605337:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00000080:00000010:1.1:1761230882.651561:0:91175:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f7a32e83. 00010000:00000010:3.1:1761230882.678873:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:3.1:1761230882.678881:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:1.1:1761230882.684890:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a694868b. 00000080:00000010:1.1:1761230882.720005:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f7a32e83. 00010000:00000010:1.1:1761230882.756812:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:1.1:1761230882.756820:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000abe282bc. 00000080:00000010:1.1:1761230882.762881:0:91178:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f7a32e83. 00000080:00000010:1.1:1761230882.798884:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f7a32e83. 00010000:00000010:1.1:1761230882.836881:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:1.1:1761230882.836893:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000019b97daf. 00010000:00000010:1.1:1761230882.873082:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:1.1:1761230882.873090:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00010000:00000010:1.1:1761230882.914191:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:1.1:1761230882.914200:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c9e10b74. 00010000:00000010:1.1:1761230882.958473:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:1.1:1761230882.958491:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:1.1:1761230882.971877:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000eb1c2166. 00010000:00000010:1.1:1761230883.035968:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761230883.035977:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:3.1:1761230883.040989:0:88987:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007c7fb311. 00010000:00000010:1.1:1761230883.078125:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:1.1:1761230883.078140:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000013f3c2e5. 00000080:00000010:3.1:1761230883.118878:0:91187:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cd8b3a27. 00000080:00000010:3.1:1761230883.154889:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004bfd5afb. 00010000:00000010:1.1:1761230883.236086:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:1.1:1761230883.236104:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000013f3c2e5. 00010000:00000010:1.1:1761230883.286888:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:1.1:1761230883.286898:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:3.1:1761230883.296266:0:91191:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000018ca2c0b. 00010000:00000010:1.1:1761230883.325909:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:1.1:1761230883.325923:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006f6778c1. 00010000:00000010:1.1:1761230883.362011:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761230883.362023:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005c7037e3. 00000080:00000010:1.1:1761230883.400055:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003788de3a. 00000080:00000010:3.1:1761230883.480915:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e1c1399. 00000080:00000010:3.1:1761230883.518890:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005df662d6. 00000080:00000010:3.1:1761230883.561017:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000926ad9c9. 00010000:00000010:1.1:1761230883.670112:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:1.1:1761230883.670137:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00010000:00000010:3.1:1761230883.713906:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:3.1:1761230883.713913:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00000080:00000010:1.1:1761230883.725031:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d50fc64a. 00000080:00000010:1.1:1761230883.763407:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000050033dab. 00010000:00000010:1.1:1761230883.783887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:1.1:1761230883.783894:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:1.1:1761230883.797895:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e9ebbe2. 00010000:00000010:1.1:1761230883.878418:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761230883.878426:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ec29a222. 00010000:00000010:1.1:1761230883.903901:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c682eeae. 00010000:00000010:1.1:1761230883.903908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:1.1:1761230883.953142:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:1.1:1761230883.953153:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:1.1:1761230883.977051:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008c696aaf. 00010000:00000010:1.1:1761230883.998090:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:1.1:1761230883.998098:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:3.1:1761230884.039771:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:3.1:1761230884.039784:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00010000:00000010:1.1:1761230884.079415:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:1.1:1761230884.079433:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:1.1:1761230884.092924:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b722c033. 00010000:00000010:1.1:1761230884.119018:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:1.1:1761230884.119036:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00010000:00000010:3.1:1761230884.150083:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:3.1:1761230884.150091:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00010000:00000010:1.1:1761230884.181297:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761230884.181308:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005c7037e3. 00010000:00000010:3.1:1761230884.238866:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761230884.238875:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:1.1:1761230884.247088:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001d587614. 00010000:00000010:3.1:1761230884.289961:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:3.1:1761230884.289968:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000967ce876. 00000080:00000010:3.1:1761230884.297886:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c540c55. 00010000:00000010:3.1:1761230884.313868:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761230884.313879:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006f6778c1. 00010000:00000010:3.1:1761230884.348985:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761230884.349004:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005170de0b. 00000080:00000010:3.1:1761230884.377013:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000db9a8b88. 00010000:00000010:1.1:1761230884.390097:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:1.1:1761230884.390113:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006f6778c1. 00010000:00000010:3.1:1761230884.447022:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:3.1:1761230884.447049:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00010000:00000010:3.1:1761230884.518554:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:3.1:1761230884.518563:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:3.1:1761230884.600986:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000bbabe4bb. 00000080:00000010:1.1:1761230884.659882:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008943c2ee. 00010000:00000010:1.1:1761230884.687900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761230884.687912:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:3.1:1761230884.692585:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000002fba8ac3. 00010000:00000010:3.1:1761230884.766880:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:3.1:1761230884.766888:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f1f326dc. 00010000:00000010:1.1:1761230884.788899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:1.1:1761230884.788915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00000080:00000010:1.1:1761230884.922452:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009a80da1e. 00000080:00000010:1.1:1761230884.960013:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009b6d58f2. 00010000:00000010:1.1:1761230884.988135:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:1.1:1761230884.988173:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00010000:00000010:1.1:1761230885.050687:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761230885.050703:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00010000:00000010:1.1:1761230885.095877:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:1.1:1761230885.095888:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:1.1:1761230885.099893:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00010000:00000010:1.1:1761230885.128639:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:1.1:1761230885.128655:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000019b97daf. 00000080:00000010:3.1:1761230885.139417:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ddac48d9. 00010000:00000010:1.1:1761230885.185974:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761230885.185983:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aafb0595. 00000080:00000010:3.1:1761230885.195766:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006a2a58ca. 00000080:00000010:1.1:1761230885.235015:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003822d66b. 00000080:00000010:1.1:1761230885.276896:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000023495250. 00010000:00000010:3.1:1761230885.310891:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:3.1:1761230885.310900:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003b5da300. 00000080:00000010:1.1:1761230885.354966:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000001ff9f0a. 00000080:00000010:1.1:1761230885.396912:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001e9392eb. 00010000:00000010:1.1:1761230885.436112:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:1.1:1761230885.436120:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:3.1:1761230885.437757:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b5a3b371. 00010000:00000010:3.1:1761230885.472727:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:3.1:1761230885.472735:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00010000:00000010:3.1:1761230885.532093:0:32:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:3.1:1761230885.532106:0:32:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00000080:00000010:3.1:1761230885.541877:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000bda996c1. 00010000:00000010:3.1:1761230885.555887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001c3ee364. 00010000:00000010:3.1:1761230885.555900:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003b5da300. 00010000:00000010:3.1:1761230885.646341:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:3.1:1761230885.646355:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00010000:00000010:3.1:1761230930.791143:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:3.1:1761230930.791156:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00000080:00000010:3.1:1761230930.814435:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009ea913bb. 00000080:00000010:1.1:1761230931.023086:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00000080:00000010:3.1:1761230931.102897:0:32:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000806c11de. 00010000:00000010:3.1:1761230931.216310:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:3.1:1761230931.216327:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:1.1:1761230931.271966:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:1.1:1761230931.271981:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00010000:00000010:3.1:1761230931.333886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d067cc47. 00010000:00000010:3.1:1761230931.333905:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00000080:00000010:1.1:1761230931.357879:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003b3ff5ca. 00000080:00000010:1.1:1761230931.433864:0:91269:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003b0bbaf4. 00010000:00000010:1.1:1761230931.557395:0:91271:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000032b67042. 00010000:00000010:1.1:1761230931.557409:0:91271:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c83761be. 00000080:00000010:3.1:1761230931.560214:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00000080:00000010:1.1:1761230931.633894:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d0098e43. 00010000:00000010:3.1:1761230931.669903:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:3.1:1761230931.669918:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00000080:00000010:1.1:1761230931.693611:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000089644b34. 00000080:00000010:1.1:1761230931.729968:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000701cb90e. 00010000:00000010:3.1:1761230931.769378:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:3.1:1761230931.769414:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003b5da300. 00000080:00000010:3.1:1761230931.786330:0:31:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c240b4ed. 00010000:00000010:1.1:1761230931.805057:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f485d354. 00010000:00000010:1.1:1761230931.805072:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003b5da300. 00010000:00000010:3.1:1761230931.857260:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:3.1:1761230931.857277:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5207efd. 00010000:00000010:1.1:1761230931.889886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:1.1:1761230931.889895:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00000080:00000010:1.1:1761230932.047897:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b0b500e8. 00000080:00000010:1.1:1761230932.101965:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b663cc8f. 00010000:00000010:1.1:1761230932.143899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:1.1:1761230932.143932:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000019b97daf. 00010000:00000010:1.1:1761230932.202247:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:1.1:1761230932.202255:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:1.1:1761230932.205986:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000de437705. 00010000:00000010:3.1:1761230932.263988:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:3.1:1761230932.264001:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006f6778c1. 00010000:00000010:3.1:1761230932.292989:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:3.1:1761230932.293003:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aaebf6f6. 00000080:00000010:1.1:1761230932.300333:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d8176c0b. 00010000:00000010:3.1:1761230932.335257:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:3.1:1761230932.335263:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00000080:00000010:3.1:1761230932.352650:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000523732a6. 00010000:00000010:3.1:1761230932.369966:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:3.1:1761230932.369974:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006f6778c1. 00010000:00000010:1.1:1761230932.417062:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:1.1:1761230932.417076:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00010000:00000010:3.1:1761230932.450071:0:88994:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:3.1:1761230932.450078:0:88994:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00010000:00000010:1.1:1761230932.493054:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:1.1:1761230932.493069:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005170de0b. 00000080:00000010:1.1:1761230932.512974:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008943c2ee. 00010000:00000010:3.1:1761230932.542869:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:3.1:1761230932.542886:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00010000:00000010:3.1:1761230976.876767:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:3.1:1761230976.876786:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:1.1:1761230976.889685:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003278ff9d. 00010000:00000010:1.1:1761230977.204931:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:1.1:1761230977.204955:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00000080:00000010:1.1:1761230977.251087:0:91307:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000daad9a21. 00000080:00000010:3.1:1761230977.443206:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000057534bdf. 00010000:00000010:1.1:1761230977.510053:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:1.1:1761230977.510061:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00000080:00000010:3.1:1761230977.577912:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000877bbbcd. 00010000:00000010:3.1:1761230977.641049:0:91313:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:3.1:1761230977.641057:0:91313:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:3.1:1761230977.673391:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e8a4e40c. 00010000:00000010:3.1:1761230977.761024:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:3.1:1761230977.761044:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000013f3c2e5. 00000080:00000010:1.1:1761230977.787404:0:88988:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e0c3f73d. 00010000:00000010:3.1:1761230977.943975:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:3.1:1761230977.943996:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:1.1:1761230978.032091:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761230978.032109:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:3.1:1761230978.044025:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000027c1817e. 00010000:00000010:3.1:1761230978.076642:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:3.1:1761230978.076667:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00000080:00000010:1.1:1761230978.109108:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a360353a. 00010000:00000010:3.1:1761230978.125885:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761230978.125920:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000abe282bc. 00000080:00000010:1.1:1761230978.140910:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b296e3a8. 00010000:00000010:3.1:1761230978.200893:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761230978.200906:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005c7037e3. 00010000:00000010:3.1:1761230978.260902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:3.1:1761230978.260919:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000720b7e17. 00010000:00000010:3.1:1761230978.324941:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:3.1:1761230978.324960:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:3.1:1761230978.340369:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000787c0332. 00010000:00000010:1.1:1761230978.394923:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:1.1:1761230978.394951:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:3.1:1761230978.431036:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e9822c4e. 00010000:00000010:1.1:1761230978.544809:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:1.1:1761230978.544831:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:1.1:1761230978.560931:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000db9a8b88. 00010000:00000010:1.1:1761230978.601900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761230978.601925:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00010000:00000010:1.1:1761230978.697895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c682eeae. 00010000:00000010:1.1:1761230978.697907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00010000:00000010:1.1:1761231024.923978:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:1.1:1761231024.923986:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:1.1:1761231024.931900:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b15036c3. 00000080:00000010:3.1:1761231024.964892:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005e438bbf. 00010000:00000010:3.1:1761231025.057960:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:3.1:1761231025.057972:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00000080:00000010:1.1:1761231025.065992:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b841249c. 00010000:00000010:3.1:1761231025.140771:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:3.1:1761231025.140785:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:1.1:1761231025.143450:0:91342:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004d107b75. 00000080:00000010:1.1:1761231025.191609:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004d107b75. 00010000:00000010:3.1:1761231025.224873:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:3.1:1761231025.224881:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00000080:00000010:1.1:1761231025.235629:0:91344:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000037a17b0d. 00010000:00000010:3.1:1761231025.278144:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:3.1:1761231025.278162:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f370345a. 00000080:00000010:1.1:1761231025.306893:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000021aeeb0a. 00010000:00000010:3.1:1761231025.327910:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:3.1:1761231025.327924:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00000080:00000010:1.1:1761231025.430872:0:91348:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000647ca513. 00010000:00000010:3.1:1761231025.474271:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761231025.474285:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:1.1:1761231025.475303:0:91349:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000647ca513. 00010000:00000010:3.1:1761231025.513894:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231025.513907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00000080:00000010:3.1:1761231025.528874:0:13:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ef36f86a. 00000080:00000010:1.1:1761231025.583302:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004d4ac0de. 00010000:00000010:3.1:1761231025.617898:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:3.1:1761231025.617913:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ad7a691b. 00000080:00000010:1.1:1761231025.630273:0:91352:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000647ca513. 00010000:00000010:1.1:1761231025.705890:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f485d354. 00010000:00000010:1.1:1761231025.705900:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006f6778c1. 00010000:00000010:1.1:1761231025.746030:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:1.1:1761231025.746045:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00010000:00000010:1.1:1761231025.778058:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:1.1:1761231025.778068:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00010000:00000010:1.1:1761231025.820898:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:1.1:1761231025.820915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000019b97daf. 00010000:00000010:3.1:1761231025.867887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:3.1:1761231025.867901:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00010000:00000010:3.1:1761231025.905488:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:3.1:1761231025.905500:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00010000:00000010:3.1:1761231025.961895:0:13:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:3.1:1761231025.961927:0:13:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c9e10b74. 00000080:00000010:3.1:1761231025.966994:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000018ca2c0b. 00010000:00000010:3.1:1761231026.032710:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001c3ee364. 00010000:00000010:3.1:1761231026.032718:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00010000:00000010:1.1:1761231026.075685:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761231026.075708:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003b5da300. 00010000:00000010:0.1:1761231026.111421:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:1.1:1761231026.126918:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f35f02da. 00010000:00000010:0.1:1761231026.155894:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:0.1:1761231026.155911:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00010000:00000010:1.1:1761231026.200012:0:88989:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:1.1:1761231026.200020:0:88989:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:3.1:1761231026.207987:0:91367:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000bda996c1. 00010000:00000010:0.1:1761231026.243893:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:0.1:1761231026.243907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00010000:00000010:3.1:1761231026.291264:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:3.1:1761231026.291430:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00010000:00000010:1.1:1761231026.362273:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:1.1:1761231026.362290:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00010000:00000010:1.1:1761231026.407425:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000032b67042. 00010000:00000010:1.1:1761231026.407448:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:0.1:1761231026.417993:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000006f2b404. 00010000:00000010:3.1:1761231026.463645:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:3.1:1761231026.463662:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c4969726. 00000080:00000010:3.1:1761231026.467925:0:91372:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c8ca1bb. 00000080:00000010:1.1:1761231026.522896:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f05b7367. 00010000:00000010:3.1:1761231026.523055:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:3.1:1761231026.523061:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c83761be. 00010000:00000010:1.1:1761231026.554898:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:1.1:1761231026.554907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:1.1:1761231026.570885:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000910e0743. 00010000:00000010:1.1:1761231026.597110:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:1.1:1761231026.597118:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:1.1:1761231026.604922:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003ace7c30. 00010000:00000010:3.1:1761231026.636010:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231026.636026:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:0.1:1761231026.655877:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000806c11de. 00000080:00000010:1.1:1761231026.720036:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c06adb11. 00010000:00000010:3.1:1761231026.749895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:3.1:1761231026.749910:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:1.1:1761231026.766901:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000beb91302. 00010000:00000010:0.1:1761231026.812902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:0.1:1761231026.812918:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00000080:00000010:3.1:1761231026.835457:0:91379:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a1c721db. 00010000:00000010:3.1:1761231026.869085:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:3.1:1761231026.869102:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00000080:00000010:1.1:1761231026.946898:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000897c7016. 00010000:00000010:0.1:1761231027.021028:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:0.1:1761231027.021048:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:3.1:1761231027.074974:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:3.1:1761231027.074987:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00010000:00000010:0.1:1761231027.110063:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:0.1:1761231027.110077:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c930e17. 00000080:00000010:1.1:1761231027.118029:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000022dc6b9b. 00000080:00000010:3.1:1761231027.168464:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003ace7c30. 00010000:00000010:3.1:1761231027.192972:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:3.1:1761231027.192989:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ff1e5574. 00000080:00000010:1.1:1761231027.198892:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000afb62743. 00000080:00000010:3.1:1761231027.236268:0:31:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000037a17b0d. 00010000:00000010:3.1:1761231027.268952:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:3.1:1761231027.268966:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00000080:00000010:1.1:1761231027.274888:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000899e612e. 00000080:00000010:0.1:1761231027.319978:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000002afcdcce. 00000080:00000010:0.1:1761231027.349970:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000027c1817e. 00000080:00000010:3.1:1761231027.396098:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e95fac7. 00010000:00000010:1.1:1761231027.459969:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:1.1:1761231027.459988:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:1.1:1761231027.474896:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004c9f6b15. 00010000:00000010:3.1:1761231027.499880:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000032b67042. 00010000:00000010:3.1:1761231027.499889:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:0.1:1761231027.505877:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000930c75c8. 00010000:00000010:3.1:1761231027.529022:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:3.1:1761231027.529030:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00010000:00000010:3.1:1761231027.556115:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001c3ee364. 00010000:00000010:3.1:1761231027.556134:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00000080:00000010:0.1:1761231027.563932:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000523732a6. 00010000:00000010:3.1:1761231027.616891:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:3.1:1761231027.616907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00010000:00000010:3.1:1761231027.657944:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:3.1:1761231027.657961:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c4969726. 00010000:00000010:3.1:1761231027.712122:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231027.712145:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ff1e5574. 00010000:00000010:3.1:1761231027.755023:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:3.1:1761231027.755045:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00000080:00000010:0.1:1761231027.766940:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000000ca4aa8. 00010000:00000010:3.1:1761231027.799022:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231027.799035:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00010000:00000010:3.1:1761231027.830936:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:3.1:1761231027.830952:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00010000:00000010:0.1:1761231027.882883:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:0.1:1761231027.882897:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00000080:00000010:3.1:1761231027.892761:0:91403:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003278ff9d. 00010000:00000010:0.1:1761231027.923899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:0.1:1761231027.923914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ff1e5574. 00000080:00000010:1.1:1761231027.935699:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003f0d8b90. 00010000:00000010:3.1:1761231027.972093:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231027.972107:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:3.1:1761231027.991721:0:91405:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003119fc9c. 00010000:00000010:3.1:1761231028.025189:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:3.1:1761231028.025205:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00000080:00000010:0.1:1761231028.039990:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000192711f4. 00010000:00000010:1.1:1761231028.083081:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:1.1:1761231028.083095:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00010000:00000010:0.1:1761231028.111819:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:0.1:1761231028.111830:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:1.1:1761231028.132617:0:91408:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000089e9938d. 00010000:00000010:1.1:1761231028.173296:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:1.1:1761231028.173306:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ff1e5574. 00000080:00000010:0.1:1761231028.184924:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000600a1cad. 00010000:00000010:3.1:1761231028.218976:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:3.1:1761231028.218990:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:1.1:1761231028.240896:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000087fe9bd1. 00010000:00000010:0.1:1761231028.266883:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231028.266891:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003b5da300. 00000080:00000010:0.1:1761231028.281411:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f259f333. 00000080:00000010:1.1:1761231028.313926:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000057ae30e1. 00010000:00000010:1.1:1761231028.344270:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:1.1:1761231028.344286:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00000080:00000010:0.1:1761231028.367800:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004d9bb60c. 00010000:00000010:1.1:1761231028.384753:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:1.1:1761231028.384770:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:0.1:1761231028.397987:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007b9a2413. 00010000:00000010:0.1:1761231028.490902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b033b999. 00010000:00000010:0.1:1761231028.490914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f2aa9f4c. 00000080:00000010:1.1:1761231028.500596:0:88992:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000050033dab. 00010000:00000010:1.1:1761231028.533008:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761231028.533021:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00010000:00000010:1.1:1761231028.585503:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231028.585514:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00000080:00000010:1.1:1761231028.594733:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ad14d849. 00000080:00000010:1.1:1761231028.639308:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f53240cb. 00010000:00000010:0.1:1761231028.675098:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:0.1:1761231028.675131:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ad7a691b. 00000080:00000010:3.1:1761231028.692072:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009b6d58f2. 00010000:00000010:0.1:1761231028.722927:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:0.1:1761231028.722945:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00000080:00000010:3.1:1761231028.777418:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008d189be2. 00010000:00000010:3.1:1761231028.808553:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:3.1:1761231028.808569:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:0.1:1761231028.831018:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f6590227. 00010000:00000010:1.1:1761231028.870942:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:1.1:1761231028.870955:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:1.1:1761231028.875095:0:91424:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f4ae988e. 00010000:00000010:3.1:1761231028.916178:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:3.1:1761231028.916194:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00010000:00000010:1.1:1761231028.956255:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f485d354. 00010000:00000010:1.1:1761231028.956274:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:0.1:1761231028.968880:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000057534bdf. 00010000:00000010:0.1:1761231029.012186:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231029.012202:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00000080:00000010:3.1:1761231029.037277:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000037a17b0d. 00010000:00000010:3.1:1761231029.070985:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:3.1:1761231029.071001:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005b9c88b6. 00000080:00000010:0.1:1761231029.092948:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b722c033. 00010000:00000010:3.1:1761231029.128884:0:88989:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:3.1:1761231029.128893:0:88989:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006f6778c1. 00010000:00000010:0.1:1761231029.204817:0:14:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:0.1:1761231029.204835:0:14:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00010000:00000010:0.1:1761231029.254917:0:91431:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:0.1:1761231029.254928:0:91431:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006f6778c1. 00000080:00000010:0.1:1761231029.262108:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000db511364. 00010000:00000010:3.1:1761231029.300955:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:3.1:1761231029.300972:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:0.1:1761231029.302896:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003119fc9c. 00010000:00000010:3.1:1761231029.349871:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:3.1:1761231029.349878:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00000080:00000010:1.1:1761231029.371297:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000021039586. 00010000:00000010:3.1:1761231029.404900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:3.1:1761231029.404913:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f370345a. 00000080:00000010:0.1:1761231029.409095:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000002925e0c1. 00010000:00000010:3.1:1761231029.439461:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:3.1:1761231029.439475:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00000080:00000010:1.1:1761231029.451887:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000023495250. 00010000:00000010:3.1:1761231029.482885:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:3.1:1761231029.482898:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005ee4f696. 00010000:00000010:0.1:1761231029.525906:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:0.1:1761231029.525915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:3.1:1761231029.592885:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:3.1:1761231029.592902:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000019b97daf. 00000080:00000010:1.1:1761231029.609933:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c6bcfad1. 00010000:00000010:3.1:1761231029.644460:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:3.1:1761231029.644469:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00000080:00000010:1.1:1761231029.651506:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b824b487. 00010000:00000010:1.1:1761231029.699892:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:1.1:1761231029.699907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:1.1:1761231029.699912:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007856f8ad. 00010000:00000010:1.1:1761231029.736913:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:1.1:1761231029.736935:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:1.1:1761231029.794928:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:1.1:1761231029.794946:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000019b97daf. 00000080:00000010:1.1:1761231029.804880:0:91442:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ac373ff. 00010000:00000010:1.1:1761231029.851478:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:1.1:1761231029.851484:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:1.1:1761231029.855060:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000057891e9. 00010000:00000010:1.1:1761231029.872935:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761231029.872949:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00010000:00000010:3.1:1761231029.957074:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:3.1:1761231029.957090:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00000080:00000010:3.1:1761231029.966234:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00010000:00000010:1.1:1761231029.999166:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:1.1:1761231029.999180:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ff1e5574. 00010000:00000010:1.1:1761231030.053341:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761231030.053361:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:3.1:1761231030.074885:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000087fe9bd1. 00010000:00000010:1.1:1761231030.116896:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:1.1:1761231030.116905:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:3.1:1761231030.143483:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:3.1:1761231030.143490:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c9e10b74. 00010000:00000010:3.1:1761231030.177125:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:3.1:1761231030.177144:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00010000:00000010:1.1:1761231075.121044:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:1.1:1761231075.121063:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00010000:00000010:1.1:1761231075.180051:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:1.1:1761231075.180070:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00010000:00000010:1.1:1761231075.237935:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:1.1:1761231075.237953:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:0.1:1761231075.241909:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f1326a00. 00010000:00000010:3.1:1761231075.268982:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:3.1:1761231075.268994:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00010000:00000010:3.1:1761231075.315179:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:3.1:1761231075.315192:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:0.1:1761231075.328934:0:12:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003788de3a. 00010000:00000010:0.1:1761231075.357928:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:0.1:1761231075.357942:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:0.1:1761231075.375905:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cee7568e. 00010000:00000010:3.1:1761231075.406225:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c682eeae. 00010000:00000010:3.1:1761231075.406234:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00000080:00000010:3.1:1761231075.416055:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000066182a37. 00010000:00000010:1.1:1761231075.441381:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:1.1:1761231075.441399:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:3.1:1761231075.504894:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:3.1:1761231075.504908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000019b97daf. 00000080:00000010:1.1:1761231075.510666:0:91466:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00010000:00000010:3.1:1761231075.556901:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:3.1:1761231075.556932:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00010000:00000010:3.1:1761231075.623130:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231075.623152:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00000080:00000010:3.1:1761231075.639890:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c610935c. 00010000:00000010:0.1:1761231075.671570:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:0.1:1761231075.671589:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:0.1:1761231075.685915:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00010000:00000010:3.1:1761231075.718909:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231075.718920:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00010000:00000010:0.1:1761231075.755372:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:0.1:1761231075.755386:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000720b7e17. 00000080:00000010:3.1:1761231075.762905:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000899e612e. 00010000:00000010:0.1:1761231075.804917:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:0.1:1761231075.804931:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00010000:00000010:1.1:1761231075.836792:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:1.1:1761231075.836809:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:0.1:1761231075.890887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:0.1:1761231075.890903:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00010000:00000010:3.1:1761231075.946901:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231075.946916:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000019b97daf. 00000080:00000010:1.1:1761231076.016943:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000bac5fb8c. 00010000:00000010:0.1:1761231076.048361:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:0.1:1761231076.048375:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00010000:00000010:0.1:1761231076.088659:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:0.1:1761231076.088669:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00000080:00000010:3.1:1761231076.097993:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a360353a. 00010000:00000010:0.1:1761231076.131800:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231076.131815:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000019b97daf. 00010000:00000010:3.1:1761231076.184484:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231076.184492:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000013f3c2e5. 00010000:00000010:3.1:1761231076.228209:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231076.228226:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00010000:00000010:1.1:1761231076.274971:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761231076.274988:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:0.1:1761231076.292364:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003e1f952c. 00010000:00000010:3.1:1761231076.331034:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:3.1:1761231076.331041:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:0.1:1761231076.347079:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006fc51f99. 00010000:00000010:1.1:1761231076.384880:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231076.384894:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00000080:00000010:1.1:1761231076.489653:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e9822c4e. 00010000:00000010:3.1:1761231076.513930:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:3.1:1761231076.513950:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000967ce876. 00000080:00000010:0.1:1761231076.517895:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000057891e9. 00000080:00000010:0.1:1761231076.549059:0:91489:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000bbabe4bb. 00010000:00000010:1.1:1761231076.579877:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:1.1:1761231076.579890:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00010000:00000010:0.1:1761231076.624899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:0.1:1761231076.624913:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bcb018ff. 00010000:00000010:1.1:1761231076.665876:0:91492:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:1.1:1761231076.665889:0:91492:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:0.1:1761231076.671083:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009b6d58f2. 00010000:00000010:0.1:1761231076.704146:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:0.1:1761231076.704155:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aaebf6f6. 00010000:00000010:0.1:1761231076.737892:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:0.1:1761231076.737908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:1.1:1761231076.751914:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003b0bbaf4. 00010000:00000010:0.1:1761231076.782902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:0.1:1761231076.782914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:0.1:1761231076.799079:0:91495:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007b9a2413. 00010000:00000010:0.1:1761231076.850375:0:91495:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231076.850384:0:91495:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004a1223b6. 00000080:00000010:0.1:1761231076.855057:0:91496:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a35479a0. 00000080:00000010:0.1:1761231076.901910:0:14:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000456339b2. 00010000:00000010:0.1:1761231076.971957:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:0.1:1761231076.971963:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5207efd. 00000080:00000010:1.1:1761231076.975186:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000db9a8b88. 00000080:00000010:1.1:1761231077.020104:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c540c55. 00000080:00000010:0.1:1761231077.076877:0:91501:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e1e65450. 00000080:00000010:3.1:1761231077.153972:0:88670:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c540c55. 00000080:00000010:0.1:1761231077.210727:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007c7fb311. 00000080:00000010:1.1:1761231077.237878:0:91504:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004c6dbadd. 00000080:00000010:3.1:1761231077.292891:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f2401a6f. 00010000:00000010:3.1:1761231077.323246:0:32:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:3.1:1761231077.323254:0:32:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00000080:00000010:1.1:1761231077.333897:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ff17c768. 00010000:00000010:3.1:1761231077.372078:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:3.1:1761231077.372085:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aafb0595. 00000080:00000010:0.1:1761231077.383657:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d159b366. 00000080:00000010:0.1:1761231077.438903:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cd8b3a27. 00000080:00000010:1.1:1761231077.496006:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000018ca2c0b. 00000080:00000010:0.1:1761231077.556908:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ff74b385. 00000080:00000010:1.1:1761231077.613099:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b6108498. 00000080:00000010:0.1:1761231077.682728:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ff74b385. 00010000:00000010:3.1:1761231077.756896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:3.1:1761231077.756910:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f1f326dc. 00000080:00000010:3.1:1761231077.773931:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c540c55. 00010000:00000010:3.1:1761231077.813094:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:3.1:1761231077.813109:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00000080:00000010:1.1:1761231077.832891:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001f8c49ba. 00010000:00000010:3.1:1761231077.863940:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:3.1:1761231077.863953:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f8fd4482. 00000080:00000010:0.1:1761231077.880967:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a7dd8716. 00010000:00000010:3.1:1761231077.919634:0:31:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f485d354. 00010000:00000010:3.1:1761231077.919655:0:31:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:1.1:1761231077.927880:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e5e1ac37. 00010000:00000010:3.1:1761231077.959660:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:3.1:1761231077.959667:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00000080:00000010:1.1:1761231077.963098:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000db9a8b88. 00010000:00000010:0.1:1761231077.999673:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:0.1:1761231077.999689:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5207efd. 00010000:00000010:3.1:1761231078.055371:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:3.1:1761231078.055378:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f8fd4482. 00010000:00000010:0.1:1761231078.095008:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:0.1:1761231078.095016:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:1.1:1761231078.105016:0:91520:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000897c7016. 00010000:00000010:0.1:1761231078.149892:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:0.1:1761231078.149911:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004a1223b6. 00000080:00000010:3.1:1761231078.165482:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000026945b88. 00010000:00000010:1.1:1761231078.192251:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:1.1:1761231078.192283:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f8fd4482. 00000080:00000010:3.1:1761231078.200978:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ddac48d9. 00010000:00000010:3.1:1761231078.236896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:3.1:1761231078.236906:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aaebf6f6. 00010000:00000010:1.1:1761231078.280886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:1.1:1761231078.280896:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:1.1:1761231078.334398:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e1808a2b. 00010000:00000010:0.1:1761231078.374427:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:0.1:1761231078.374435:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:1.1:1761231078.377975:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008d189be2. 00000080:00000010:1.1:1761231078.496895:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000066182a37. 00010000:00000010:0.1:1761231078.529931:0:88995:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:0.1:1761231078.529939:0:88995:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000967ce876. 00000080:00000010:1.1:1761231078.545882:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000088a4593e. 00010000:00000010:0.1:1761231078.579889:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001c3ee364. 00010000:00000010:0.1:1761231078.579898:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:1.1:1761231078.602898:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000701cb90e. 00010000:00000010:0.1:1761231078.628895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:0.1:1761231078.628924:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:1.1:1761231078.640089:0:91532:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004d107b75. 00000080:00000010:1.1:1761231078.744272:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000647ca513. 00010000:00000010:1.1:1761231078.771287:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001c3ee364. 00010000:00000010:1.1:1761231078.771301:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00000080:00000010:1.1:1761231078.836971:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000926ad9c9. 00010000:00000010:0.1:1761231078.877751:0:91537:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:0.1:1761231078.877761:0:91537:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000720b7e17. 00000080:00000010:0.1:1761231078.888952:0:91537:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000057891e9. 00010000:00000010:1.1:1761231078.918976:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:1.1:1761231078.918984:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:0.1:1761231078.921156:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000027c1817e. 00010000:00000010:1.1:1761231078.950891:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d067cc47. 00010000:00000010:1.1:1761231078.950916:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00000080:00000010:1.1:1761231078.960894:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000de437705. 00000080:00000010:1.1:1761231079.085928:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:3.1:1761231079.137910:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:3.1:1761231079.137929:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:0.1:1761231079.148025:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000877bbbcd. 00010000:00000010:3.1:1761231079.182961:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:3.1:1761231079.182972:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00000080:00000010:0.1:1761231079.184960:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e9822c4e. 00000080:00000010:1.1:1761231079.219883:0:91545:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b0b500e8. 00000080:00000010:1.1:1761231079.276039:0:91546:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b0b500e8. 00010000:00000010:0.1:1761231079.376890:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:0.1:1761231079.376913:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000af5fa112. 00000080:00000010:0.1:1761231079.391491:0:91548:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:0.1:1761231079.442288:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001c3ee364. 00010000:00000010:0.1:1761231079.442297:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:1.1:1761231079.463926:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ff17c768. 00010000:00000010:0.1:1761231079.510049:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:0.1:1761231079.510065:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f370345a. 00000080:00000010:0.1:1761231079.514889:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d1785496. 00000080:00000010:1.1:1761231079.555956:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000089e9938d. 00000080:00000010:1.1:1761231079.589904:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004c6dbadd. 00010000:00000010:1.1:1761231079.617283:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:1.1:1761231079.617290:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00000080:00000010:0.1:1761231079.633904:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000006de6731. 00010000:00000010:3.1:1761231079.670087:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:3.1:1761231079.670103:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00000080:00000010:0.1:1761231079.690292:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000084e32c56. 00010000:00000010:1.1:1761231079.744279:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:1.1:1761231079.744292:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005b9c88b6. 00000080:00000010:1.1:1761231079.753311:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006a2a58ca. 00010000:00000010:1.1:1761231079.809917:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:1.1:1761231079.809934:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:1.1:1761231079.817941:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000456339b2. 00010000:00000010:1.1:1761231079.847900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761231079.847915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ec280f6b. 00000080:00000010:0.1:1761231079.857204:0:91557:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000afb62743. 00010000:00000010:1.1:1761231079.903781:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:1.1:1761231079.903793:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:1.1:1761231079.945905:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761231079.945915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:3.1:1761231079.978908:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:3.1:1761231079.978921:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00010000:00000010:1.1:1761231080.041012:0:91561:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761231080.041022:0:91561:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00010000:00000010:3.1:1761231080.074298:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:3.1:1761231080.074307:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005b9c88b6. 00000080:00000010:1.1:1761231080.085657:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c9357289. 00010000:00000010:1.1:1761231080.135306:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:1.1:1761231080.135324:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a2699cea. 00000080:00000010:3.1:1761231080.155880:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003f0d8b90. 00010000:00000010:3.1:1761231080.203899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:3.1:1761231080.203919:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:3.1:1761231080.262936:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:3.1:1761231080.262953:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00010000:00000010:1.1:1761231080.311110:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f485d354. 00010000:00000010:1.1:1761231080.311126:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:0.1:1761231080.319912:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e9822c4e. 00010000:00000010:1.1:1761231080.342113:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:1.1:1761231080.342127:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00000080:00000010:1.1:1761231080.361581:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b5773b23. 00010000:00000010:3.1:1761231080.390906:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:3.1:1761231080.390926:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:3.1:1761231080.411668:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000beb91302. 00010000:00000010:0.1:1761231080.436440:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:0.1:1761231080.436456:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:1.1:1761231080.459439:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000090aa726d. 00010000:00000010:3.1:1761231080.485112:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:3.1:1761231080.485126:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:0.1:1761231080.512304:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d57469c8. 00010000:00000010:3.1:1761231080.576352:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:3.1:1761231080.576369:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ad7a691b. 00010000:00000010:3.1:1761231080.624001:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231080.624036:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00000080:00000010:3.1:1761231080.648527:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00010000:00000010:3.1:1761231080.689349:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:3.1:1761231080.689363:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00010000:00000010:1.1:1761231080.770627:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:1.1:1761231080.770634:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f2aa9f4c. 00000080:00000010:0.1:1761231080.783731:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000054afda06. 00010000:00000010:3.1:1761231080.818960:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:3.1:1761231080.818969:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:3.1:1761231080.841899:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007c7fb311. 00010000:00000010:1.1:1761231080.848611:0:88989:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:1.1:1761231080.848623:0:88989:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:0.1:1761231080.857593:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003119fc9c. 00010000:00000010:0.1:1761231080.886894:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231080.886904:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00010000:00000010:1.1:1761231080.931139:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231080.931155:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00010000:00000010:3.1:1761231080.979964:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:3.1:1761231080.979989:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00000080:00000010:1.1:1761231081.029974:0:91580:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000084e32c56. 00010000:00000010:0.1:1761231081.066084:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:0.1:1761231081.066101:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c4969726. 00000080:00000010:0.1:1761231081.083900:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000084e32c56. 00010000:00000010:1.1:1761231081.110000:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:1.1:1761231081.110007:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:1.1:1761231081.117914:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000089644b34. 00010000:00000010:1.1:1761231081.148908:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:1.1:1761231081.148916:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00000080:00000010:0.1:1761231081.204891:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009ba4f828. 00010000:00000010:3.1:1761231081.228430:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761231081.228451:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:1.1:1761231081.235877:0:91585:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009a80da1e. 00010000:00000010:0.1:1761231081.275066:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:0.1:1761231081.275087:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c930e17. 00000080:00000010:0.1:1761231081.289900:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009a80da1e. 00010000:00000010:0.1:1761231081.333059:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:0.1:1761231081.333072:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:0.1:1761231081.337879:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000523732a6. 00010000:00000010:0.1:1761231081.387894:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:0.1:1761231081.387908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00000080:00000010:0.1:1761231081.401164:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007b1d315f. 00010000:00000010:0.1:1761231081.446720:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:0.1:1761231081.446738:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00010000:00000010:1.1:1761231081.502713:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231081.502725:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:0.1:1761231081.510505:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b841249c. 00010000:00000010:0.1:1761231081.545896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:0.1:1761231081.545910:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00010000:00000010:1.1:1761231081.587893:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:1.1:1761231081.587907:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00010000:00000010:0.1:1761231081.645363:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c682eeae. 00010000:00000010:0.1:1761231081.645377:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c930e17. 00000080:00000010:0.1:1761231081.651955:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000654599ee. 00010000:00000010:1.1:1761231081.688593:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761231081.688602:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00000080:00000010:1.1:1761231081.710186:0:91594:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cce09f8d. 00010000:00000010:1.1:1761231081.738888:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:1.1:1761231081.738897:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:0.1:1761231081.754473:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006de9a0c1. 00010000:00000010:1.1:1761231081.782125:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:1.1:1761231081.782134:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:0.1:1761231081.833072:0:91597:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005df662d6. 00010000:00000010:0.1:1761231081.883727:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:0.1:1761231081.883743:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:0.1:1761231081.903914:0:88989:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a1c721db. 00000080:00000010:3.1:1761231081.936912:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000009663ed. 00010000:00000010:1.1:1761231081.938380:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:1.1:1761231081.938918:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00010000:00000010:0.1:1761231081.964999:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:0.1:1761231081.965013:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:0.1:1761231081.991936:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e1c1399. 00010000:00000010:0.1:1761231082.016979:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:0.1:1761231082.016993:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00010000:00000010:0.1:1761231082.067708:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:0.1:1761231082.067715:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:0.1:1761231082.086252:0:1356:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000f2006b8. 00010000:00000010:1.1:1761231082.107936:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:1.1:1761231082.107952:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:0.1:1761231082.143873:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:0.1:1761231082.143896:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c4969726. 00000080:00000010:3.1:1761231082.154075:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001174e0d2. 00000080:00000010:0.1:1761231082.194937:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b2aa4cab. 00010000:00000010:0.1:1761231082.231996:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:0.1:1761231082.232013:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00000080:00000010:1.1:1761231082.242052:0:91606:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000066182a37. 00000080:00000010:3.1:1761231082.292954:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e3e436ae. 00010000:00000010:0.1:1761231082.320990:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004807df18. 00010000:00000010:0.1:1761231082.321004:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:1.1:1761231082.329955:0:91608:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000055b04bfa. 00010000:00000010:3.1:1761231082.362134:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:3.1:1761231082.362153:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ad7a691b. 00000080:00000010:0.1:1761231082.368913:0:91609:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b296e3a8. 00010000:00000010:0.1:1761231082.449907:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:0.1:1761231082.449921:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00010000:00000010:0.1:1761231082.504138:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:0.1:1761231082.504149:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00000080:00000010:0.1:1761231082.516291:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000087fe9bd1. 00010000:00000010:3.1:1761231082.545147:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:3.1:1761231082.545155:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:1.1:1761231082.558947:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e0c3f73d. 00010000:00000010:0.1:1761231082.593903:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:0.1:1761231082.593916:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:0.1:1761231082.599641:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00010000:00000010:3.1:1761231082.630917:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231082.630941:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00010000:00000010:0.1:1761231082.701012:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:0.1:1761231082.701026:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00010000:00000010:3.1:1761231082.755553:0:91617:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231082.755571:0:91617:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:3.1:1761231082.765905:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000de437705. 00010000:00000010:0.1:1761231082.836301:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:0.1:1761231082.836317:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00010000:00000010:1.1:1761231082.866886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:1.1:1761231082.866899:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00010000:00000010:0.1:1761231082.909889:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:0.1:1761231082.909907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00000080:00000010:3.1:1761231082.927111:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000002fba8ac3. 00010000:00000010:0.1:1761231082.976250:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:0.1:1761231082.976263:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f85cd001. 00000080:00000010:0.1:1761231082.980041:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d6a3007a. 00010000:00000010:0.1:1761231083.011885:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231083.011901:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000073fe992b. 00000080:00000010:3.1:1761231083.024993:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d7db8d9f. 00010000:00000010:0.1:1761231083.055058:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:0.1:1761231083.055088:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00010000:00000010:0.1:1761231083.102895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:0.1:1761231083.102914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aa24da73. 00000080:00000010:1.1:1761231083.125568:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:0.1:1761231083.165420:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:0.1:1761231083.165440:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005b9c88b6. 00000080:00000010:1.1:1761231083.184160:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000021039586. 00010000:00000010:3.1:1761231083.199899:0:32:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231083.199915:0:32:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00010000:00000010:0.1:1761231083.256025:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:0.1:1761231083.256046:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00000080:00000010:1.1:1761231083.263889:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004c6dbadd. 00010000:00000010:1.1:1761231083.305921:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:1.1:1761231083.305937:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00000080:00000010:0.1:1761231083.317027:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b5773b23. 00010000:00000010:1.1:1761231083.345223:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:1.1:1761231083.345231:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00000080:00000010:1.1:1761231083.356994:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e1808a2b. 00010000:00000010:1.1:1761231083.393934:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:1.1:1761231083.393947:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00010000:00000010:0.1:1761231083.446897:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:0.1:1761231083.446913:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00000080:00000010:0.1:1761231083.468100:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ad14d849. 00010000:00000010:1.1:1761231083.486519:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:1.1:1761231083.486537:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00010000:00000010:3.1:1761231083.533080:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231083.533095:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00000080:00000010:3.1:1761231083.543961:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a20efd9d. 00010000:00000010:1.1:1761231083.576719:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:1.1:1761231083.576733:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00000080:00000010:1.1:1761231083.594995:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c2ff289. 00010000:00000010:0.1:1761231083.636927:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:0.1:1761231083.636944:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f370345a. 00000080:00000010:0.1:1761231083.642885:0:91636:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000057891e9. 00010000:00000010:0.1:1761231083.690401:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:0.1:1761231083.690415:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:0.1:1761231083.699885:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003696a3b3. 00010000:00000010:3.1:1761231083.728900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:3.1:1761231083.728914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:1.1:1761231083.737437:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000926ad9c9. 00010000:00000010:0.1:1761231083.786207:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:0.1:1761231083.786223:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00010000:00000010:3.1:1761231083.838984:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231083.839002:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b51c1403. 00000080:00000010:0.1:1761231083.842924:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b865dd43. 00010000:00000010:0.1:1761231083.895959:0:88989:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:0.1:1761231083.895983:0:88989:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005ee4f696. 00010000:00000010:0.1:1761231083.959071:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:0.1:1761231083.959078:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00000080:00000010:0.1:1761231083.959966:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000066182a37. 00010000:00000010:1.1:1761231083.990898:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761231083.990916:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:0.1:1761231084.052065:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:0.1:1761231084.052073:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b51c1403. 00000080:00000010:0.1:1761231084.069029:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000dec5607e. 00010000:00000010:0.1:1761231084.117885:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:0.1:1761231084.117897:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00000080:00000010:1.1:1761231084.126905:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000000ca4aa8. 00010000:00000010:0.1:1761231084.164401:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:0.1:1761231084.164414:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:1.1:1761231084.175192:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000006de6731. 00000080:00000010:3.1:1761231084.224890:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000089644b34. 00010000:00000010:0.1:1761231084.267894:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:0.1:1761231084.267931:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:3.1:1761231084.290983:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f35f02da. 00010000:00000010:3.1:1761231084.384203:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:3.1:1761231084.384211:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00000080:00000010:0.1:1761231084.390127:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000023495250. 00010000:00000010:0.1:1761231084.418015:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:0.1:1761231084.418029:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00000080:00000010:3.1:1761231084.423184:0:91651:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c06adb11. 00010000:00000010:0.1:1761231084.465898:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:0.1:1761231084.465918:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000810266a2. 00010000:00000010:1.1:1761231084.522896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:1.1:1761231084.522913:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00000080:00000010:3.1:1761231084.535868:0:91653:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e9822c4e. 00010000:00000010:0.1:1761231084.579935:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:0.1:1761231084.579956:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00010000:00000010:1.1:1761231084.669900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761231084.669916:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:1.1:1761231084.714015:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fd1cf851. 00000080:00000010:1.1:1761231084.757221:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000025efff4a. 00000080:00000010:0.1:1761231084.793883:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000082515979. 00010000:00000010:1.1:1761231084.822146:0:91660:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761231084.822160:0:91660:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:1.1:1761231084.834215:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e80b1bdc. 00000080:00000010:1.1:1761231084.914310:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000806c11de. 00000080:00000010:3.1:1761231085.002237:0:88989:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000dec5607e. 00010000:00000010:3.1:1761231085.037896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:3.1:1761231085.037912:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:0.1:1761231085.047892:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000f2006b8. 00010000:00000010:0.1:1761231085.077713:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:0.1:1761231085.077727:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:3.1:1761231085.117994:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007115da5c. 00010000:00000010:0.1:1761231085.143515:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:0.1:1761231085.143525:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00000080:00000010:0.1:1761231085.148136:0:91668:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006ae998d0. 00000080:00000010:1.1:1761231085.214196:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a746fca4. 00010000:00000010:0.1:1761231085.248891:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:0.1:1761231085.248908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000967ce876. 00000080:00000010:1.1:1761231085.272705:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000899e612e. 00010000:00000010:3.1:1761231085.307893:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:3.1:1761231085.307907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:0.1:1761231085.312986:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000087fe9bd1. 00000080:00000010:0.1:1761231085.353965:0:91672:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000192711f4. 00010000:00000010:3.1:1761231085.393133:0:32:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:3.1:1761231085.393153:0:32:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00000080:00000010:1.1:1761231085.415896:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cce09f8d. 00000080:00000010:1.1:1761231085.461960:0:91674:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f1326a00. 00010000:00000010:0.1:1761231085.493375:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:0.1:1761231085.493397:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aaebf6f6. 00000080:00000010:1.1:1761231085.512093:0:91675:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f1326a00. 00010000:00000010:0.1:1761231085.532912:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:0.1:1761231085.532917:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004a1223b6. 00000080:00000010:0.1:1761231085.540907:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ba19c22. 00010000:00000010:0.1:1761231085.564895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:0.1:1761231085.564914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011645181. 00000080:00000010:3.1:1761231085.572069:0:91677:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000055b04bfa. 00010000:00000010:1.1:1761231085.605890:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:1.1:1761231085.605899:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00000080:00000010:0.1:1761231085.621917:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000055b04bfa. 00010000:00000010:1.1:1761231085.643921:0:91679:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:1.1:1761231085.643936:0:91679:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00000080:00000010:0.1:1761231085.707500:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000044aec3ee. 00010000:00000010:1.1:1761231085.728100:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761231085.728112:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00000080:00000010:3.1:1761231085.735889:0:91681:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:0.1:1761231085.773270:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:0.1:1761231085.773287:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011645181. 00000080:00000010:3.1:1761231085.785017:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:1.1:1761231085.812888:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761231085.812904:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00000080:00000010:0.1:1761231085.830968:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001f8c49ba. 00010000:00000010:1.1:1761231085.869957:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:1.1:1761231085.869971:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00010000:00000010:1.1:1761231085.926923:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:1.1:1761231085.926932:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00000080:00000010:0.1:1761231085.933118:0:12:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000037a17b0d. 00010000:00000010:1.1:1761231085.968614:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:1.1:1761231085.968637:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:1.1:1761231085.980911:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004c6dbadd. 00010000:00000010:1.1:1761231086.016025:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:1.1:1761231086.016038:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:1.1:1761231086.061406:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761231086.077806:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00000080:00000010:0.1:1761231086.114879:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000084e32c56. 00010000:00000010:1.1:1761231086.152960:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:1.1:1761231086.152978:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00000080:00000010:0.1:1761231086.161884:0:91690:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004f293f01. 00000080:00000010:0.1:1761231086.206754:0:91691:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007c7fb311. 00010000:00000010:3.1:1761231086.247050:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:3.1:1761231086.247067:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:1.1:1761231086.255990:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000062b3a65c. 00010000:00000010:3.1:1761231086.288145:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:3.1:1761231086.288162:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f1f326dc. 00000080:00000010:1.1:1761231086.298988:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000ca65ad5. 00010000:00000010:3.1:1761231086.332150:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:3.1:1761231086.332168:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00010000:00000010:1.1:1761231086.407483:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:1.1:1761231086.407496:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:0.1:1761231086.415016:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f53240cb. 00010000:00000010:1.1:1761231086.456897:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:1.1:1761231086.456938:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aacb42a. 00000080:00000010:3.1:1761231086.473880:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00010000:00000010:0.1:1761231086.513907:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:0.1:1761231086.513927:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aafb0595. 00010000:00000010:1.1:1761231086.572897:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001c3ee364. 00010000:00000010:1.1:1761231086.572914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:0.1:1761231086.597042:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009ba4f828. 00010000:00000010:1.1:1761231086.653949:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231086.653965:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aafb0595. 00010000:00000010:3.1:1761231086.729951:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:3.1:1761231086.729960:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00010000:00000010:3.1:1761231086.792014:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000032b67042. 00010000:00000010:3.1:1761231086.792034:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:0.1:1761231086.817189:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000090aa726d. 00010000:00000010:3.1:1761231086.848052:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004807df18. 00010000:00000010:3.1:1761231086.848070:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aafb0595. 00000080:00000010:1.1:1761231086.870894:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e991364b. 00010000:00000010:3.1:1761231086.899433:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:3.1:1761231086.899449:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:1.1:1761231086.948041:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:1.1:1761231086.948061:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00010000:00000010:3.1:1761231086.992320:0:88995:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d067cc47. 00010000:00000010:3.1:1761231086.992327:0:88995:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003eca7fc5. 00000080:00000010:0.1:1761231087.003184:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a7dd8716. 00010000:00000010:3.1:1761231087.033519:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d067cc47. 00010000:00000010:3.1:1761231087.033535:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00000080:00000010:3.1:1761231087.052884:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f35f02da. 00010000:00000010:0.1:1761231087.104888:0:88989:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:0.1:1761231087.104895:0:88989:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f1f326dc. 00000080:00000010:1.1:1761231087.112009:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000079b4f4f9. 00010000:00000010:3.1:1761231087.149904:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761231087.149922:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00000080:00000010:1.1:1761231087.163664:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c60b71f. 00010000:00000010:3.1:1761231087.206709:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:3.1:1761231087.206726:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:1.1:1761231087.230027:0:91710:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003119fc9c. 00010000:00000010:1.1:1761231087.269372:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:1.1:1761231087.269382:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5207efd. 00000080:00000010:1.1:1761231087.272983:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000079b4f4f9. 00010000:00000010:0.1:1761231087.296906:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:0.1:1761231087.296926:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00000080:00000010:3.1:1761231087.307316:0:91712:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f5313e06. 00010000:00000010:0.1:1761231087.349732:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:0.1:1761231087.349748:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00000080:00000010:1.1:1761231087.369927:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b5773b23. 00010000:00000010:3.1:1761231087.399922:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231087.399940:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00010000:00000010:3.1:1761231087.442790:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:3.1:1761231087.442804:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00010000:00000010:3.1:1761231087.487440:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:3.1:1761231087.487449:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00000080:00000010:1.1:1761231087.498397:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ad14d849. 00010000:00000010:0.1:1761231087.530981:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:0.1:1761231087.530997:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5207efd. 00000080:00000010:1.1:1761231087.542057:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f2401a6f. 00010000:00000010:3.1:1761231087.580889:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:3.1:1761231087.580908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00010000:00000010:0.1:1761231087.631862:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:0.1:1761231087.631875:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:0.1:1761231087.681955:0:91720:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:0.1:1761231087.681966:0:91720:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00000080:00000010:0.1:1761231087.689009:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e9822c4e. 00010000:00000010:0.1:1761231087.723085:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:0.1:1761231087.723104:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00010000:00000010:0.1:1761231087.771902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:0.1:1761231087.771918:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004a1223b6. 00000080:00000010:3.1:1761231087.777786:0:88989:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c540c55. 00010000:00000010:0.1:1761231087.801949:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:0.1:1761231087.801965:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003eca7fc5. 00010000:00000010:0.1:1761231087.840893:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:0.1:1761231087.840901:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aaebf6f6. 00010000:00000010:0.1:1761231087.885153:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:0.1:1761231087.885169:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005170de0b. 00000080:00000010:1.1:1761231087.898894:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007874886b. 00010000:00000010:1.1:1761231088.006907:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:1.1:1761231088.006922:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:3.1:1761231088.026129:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000dec5607e. 00010000:00000010:1.1:1761231088.053505:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:1.1:1761231088.053520:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000967ce876. 00000080:00000010:0.1:1761231088.065194:0:91728:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e991364b. 00010000:00000010:1.1:1761231088.099906:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:1.1:1761231088.099921:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000967ce876. 00010000:00000010:1.1:1761231088.140516:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000032b67042. 00010000:00000010:1.1:1761231088.140533:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:0.1:1761231088.146887:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000027c1817e. 00000080:00000010:1.1:1761231088.202970:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ad14d849. 00010000:00000010:1.1:1761231088.233053:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:1.1:1761231088.233069:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00010000:00000010:3.1:1761231088.275293:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231088.275306:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000810266a2. 00010000:00000010:1.1:1761231088.337169:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231088.337190:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00010000:00000010:3.1:1761231088.396903:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:3.1:1761231088.396928:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:1.1:1761231088.416957:0:91735:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f5313e06. 00010000:00000010:3.1:1761231088.455721:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761231088.455746:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000967ce876. 00000080:00000010:1.1:1761231088.476030:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006fc51f99. 00010000:00000010:0.1:1761231088.539899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:0.1:1761231088.539917:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:0.1:1761231088.559923:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009b7c375e. 00010000:00000010:0.1:1761231088.608920:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:0.1:1761231088.608945:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00010000:00000010:1.1:1761231088.652066:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:1.1:1761231088.652091:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00010000:00000010:3.1:1761231088.712725:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004807df18. 00010000:00000010:3.1:1761231088.712747:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000013f3c2e5. 00000080:00000010:1.1:1761231088.734968:0:91740:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003cfab26f. 00010000:00000010:1.1:1761231088.771070:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:1.1:1761231088.771088:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000abe282bc. 00000080:00000010:1.1:1761231088.775932:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ae34632. 00000080:00000010:1.1:1761231088.822080:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000006f2b404. 00000080:00000010:1.1:1761231088.913863:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000062b3a65c. 00010000:00000010:0.1:1761231088.964360:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:0.1:1761231088.964382:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00000080:00000010:0.1:1761231088.982333:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000018ca2c0b. 00010000:00000010:3.1:1761231089.012608:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:3.1:1761231089.012633:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:0.1:1761231089.021921:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c2ff289. 00010000:00000010:0.1:1761231089.061967:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:0.1:1761231089.061977:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00010000:00000010:3.1:1761231089.122182:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231089.122206:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000810266a2. 00000080:00000010:3.1:1761231089.159319:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000055b04bfa. 00000080:00000010:1.1:1761231089.201021:0:91748:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b0b500e8. 00010000:00000010:3.1:1761231089.244026:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:3.1:1761231089.244048:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000720b7e17. 00000080:00000010:1.1:1761231089.257000:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b0b500e8. 00010000:00000010:3.1:1761231089.308187:0:91749:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231089.308200:0:91749:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00000080:00000010:3.1:1761231089.325891:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00010000:00000010:0.1:1761231089.370107:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:0.1:1761231089.370125:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00000080:00000010:1.1:1761231089.379895:0:91751:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c610935c. 00010000:00000010:3.1:1761231089.437048:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:3.1:1761231089.437074:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:0.1:1761231089.450934:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000006de6731. 00010000:00000010:0.1:1761231089.497249:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:0.1:1761231089.497269:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:3.1:1761231089.549896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:3.1:1761231089.549915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000810266a2. 00000080:00000010:1.1:1761231089.572899:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:3.1:1761231089.617973:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:3.1:1761231089.617997:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00010000:00000010:3.1:1761231089.662995:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231089.663019:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:3.1:1761231089.680941:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cee7568e. 00010000:00000010:3.1:1761231089.715880:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:3.1:1761231089.715888:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005ee4f696. 00000080:00000010:1.1:1761231089.730257:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000057185856. 00010000:00000010:1.1:1761231089.752629:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:1.1:1761231089.752635:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:0.1:1761231089.765887:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000317a378. 00010000:00000010:3.1:1761231089.791880:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:3.1:1761231089.791886:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f370345a. 00010000:00000010:0.1:1761231089.833212:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:0.1:1761231089.833226:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00010000:00000010:0.1:1761231089.885493:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:0.1:1761231089.885512:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00010000:00000010:0.1:1761231089.937920:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:0.1:1761231089.937932:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:0.1:1761231089.956903:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e9822c4e. 00010000:00000010:0.1:1761231089.987887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:0.1:1761231089.987896:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fde8dacb. 00000080:00000010:1.1:1761231089.994224:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003e1f952c. 00010000:00000010:0.1:1761231090.026968:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b033b999. 00010000:00000010:0.1:1761231090.026976:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005b9c88b6. 00000080:00000010:1.1:1761231090.032869:0:91764:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ba19c22. 00010000:00000010:3.1:1761231090.073085:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b033b999. 00010000:00000010:3.1:1761231090.073093:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:0.1:1761231090.074895:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ba19c22. 00010000:00000010:3.1:1761231090.149503:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231090.149517:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000810266a2. 00000080:00000010:1.1:1761231090.153949:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c6bcfad1. 00010000:00000010:0.1:1761231090.179937:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f485d354. 00010000:00000010:0.1:1761231090.179954:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00010000:00000010:1.1:1761231090.210132:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:1.1:1761231090.210148:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00010000:00000010:1.1:1761231090.328657:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:1.1:1761231090.328672:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00000080:00000010:0.1:1761231090.331936:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cd8b3a27. 00010000:00000010:3.1:1761231090.371611:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:3.1:1761231090.371626:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00000080:00000010:0.1:1761231090.383252:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003278ff9d. 00010000:00000010:3.1:1761231090.461931:0:91775:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231090.461945:0:91775:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000810266a2. 00000080:00000010:3.1:1761231090.470887:0:1356:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001444fa72. 00010000:00000010:1.1:1761231090.544890:0:91776:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:1.1:1761231090.544897:0:91776:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00000080:00000010:1.1:1761231090.549104:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b663cc8f. 00010000:00000010:3.1:1761231090.581965:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:3.1:1761231090.581974:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ad7a691b. 00000080:00000010:3.1:1761231090.592141:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ae34632. 00010000:00000010:1.1:1761231090.617099:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:1.1:1761231090.617115:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:0.1:1761231090.622885:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cce09f8d. 00010000:00000010:3.1:1761231090.649975:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:3.1:1761231090.650007:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00010000:00000010:3.1:1761231090.705289:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:3.1:1761231090.705308:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00010000:00000010:0.1:1761231090.775886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231090.775893:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000810266a2. 00010000:00000010:3.1:1761231090.818714:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:3.1:1761231090.818736:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00010000:00000010:3.1:1761231090.857373:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231090.857388:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f2aa9f4c. 00000080:00000010:1.1:1761231090.879545:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000aba8d18d. 00010000:00000010:3.1:1761231090.903953:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:3.1:1761231090.903966:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aa24da73. 00000080:00000010:1.1:1761231090.909336:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005df662d6. 00010000:00000010:1.1:1761231090.954264:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761231090.954280:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00010000:00000010:1.1:1761231091.016373:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:1.1:1761231091.016390:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00000080:00000010:3.1:1761231091.018932:0:91787:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000640e53cd. 00010000:00000010:1.1:1761231091.092007:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:1.1:1761231091.092037:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aa24da73. 00000080:00000010:0.1:1761231091.123883:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000dec5607e. 00000080:00000010:0.1:1761231091.217071:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000037a17b0d. 00010000:00000010:3.1:1761231091.217802:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:3.1:1761231091.217811:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00010000:00000010:0.1:1761231091.262892:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:0.1:1761231091.262913:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00010000:00000010:0.1:1761231091.317905:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:0.1:1761231091.317923:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00000080:00000010:1.1:1761231091.391129:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fd17af8f. 00010000:00000010:0.1:1761231091.392613:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:0.1:1761231091.392623:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00010000:00000010:0.1:1761231091.414702:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:0.1:1761231091.414726:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00010000:00000010:0.1:1761231091.468110:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:0.1:1761231091.468124:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f85cd001. 00010000:00000010:0.1:1761231091.503528:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:0.1:1761231091.503545:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c4969726. 00000080:00000010:1.1:1761231091.524162:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005bb06bbd. 00010000:00000010:0.1:1761231091.561465:0:91797:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:0.1:1761231091.561484:0:91797:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:0.1:1761231091.569945:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007c7fb311. 00010000:00000010:3.1:1761231091.601916:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:3.1:1761231091.601929:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:1.1:1761231091.610883:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000062b3a65c. 00010000:00000010:3.1:1761231091.639375:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:3.1:1761231091.639384:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00000080:00000010:0.1:1761231091.646082:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005ecf7799. 00010000:00000010:0.1:1761231091.693138:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:0.1:1761231091.693148:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00000080:00000010:1.1:1761231091.700979:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000037a17b0d. 00010000:00000010:0.1:1761231091.727901:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:0.1:1761231091.727909:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:0.1:1761231091.746917:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000027a3870d. 00000080:00000010:1.1:1761231091.797976:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d159b366. 00010000:00000010:3.1:1761231091.825890:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:3.1:1761231091.825905:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00010000:00000010:1.1:1761231091.896677:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c682eeae. 00010000:00000010:1.1:1761231091.896692:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:0.1:1761231091.914889:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009b6d58f2. 00010000:00000010:3.1:1761231091.961013:0:88995:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:3.1:1761231091.961020:0:88995:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:1.1:1761231091.973999:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f03444bf. 00010000:00000010:1.1:1761231091.997907:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:1.1:1761231091.997928:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00000080:00000010:1.1:1761231092.009231:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000002b56de85. 00010000:00000010:1.1:1761231092.039054:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:1.1:1761231092.039073:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00010000:00000010:1.1:1761231092.095950:0:91808:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:1.1:1761231092.095963:0:91808:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:1.1:1761231092.100915:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d7db8d9f. 00010000:00000010:1.1:1761231092.137915:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761231092.137927:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00000080:00000010:0.1:1761231092.145907:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009b6d58f2. 00010000:00000010:1.1:1761231092.176896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:1.1:1761231092.176914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:0.1:1761231092.198894:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f35f02da. 00010000:00000010:1.1:1761231092.228031:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:1.1:1761231092.228038:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00000080:00000010:0.1:1761231092.241901:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004d107b75. 00010000:00000010:1.1:1761231092.267021:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:1.1:1761231092.267036:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:1.1:1761231092.316691:0:91813:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003696a3b3. 00010000:00000010:1.1:1761231092.353968:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761231092.353985:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00000080:00000010:0.1:1761231092.371031:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000057891e9. 00000080:00000010:1.1:1761231092.411617:0:91815:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b0b500e8. 00010000:00000010:3.1:1761231092.453317:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:3.1:1761231092.453326:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00010000:00000010:1.1:1761231092.512611:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:1.1:1761231092.512628:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:1.1:1761231092.539211:0:91817:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f53240cb. 00000080:00000010:3.1:1761231092.656015:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e9822c4e. 00010000:00000010:1.1:1761231092.677873:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:1.1:1761231092.677888:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00010000:00000010:3.1:1761231092.716998:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:3.1:1761231092.717012:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00010000:00000010:1.1:1761231092.781923:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761231092.781935:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00000080:00000010:0.1:1761231092.796395:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004f293f01. 00010000:00000010:3.1:1761231092.830417:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:3.1:1761231092.830433:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:0.1:1761231092.839060:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000877bbbcd. 00010000:00000010:0.1:1761231092.870891:0:91823:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:0.1:1761231092.870898:0:91823:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00000080:00000010:0.1:1761231092.883327:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000546aef9. 00010000:00000010:0.1:1761231092.914479:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:0.1:1761231092.914489:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c4969726. 00000080:00000010:1.1:1761231092.928954:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e95fac7. 00010000:00000010:0.1:1761231092.956724:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:0.1:1761231092.956732:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00000080:00000010:3.1:1761231092.964883:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000057534bdf. 00010000:00000010:3.1:1761231093.047887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231093.047898:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00000080:00000010:3.1:1761231093.051676:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004c6dbadd. 00010000:00000010:0.1:1761231093.188968:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:0.1:1761231093.188981:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ad7a691b. 00000080:00000010:1.1:1761231093.194665:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008ac349b2. 00010000:00000010:0.1:1761231093.270490:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:0.1:1761231093.270509:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:1.1:1761231093.280102:0:91833:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000016fd0afe. 00000080:00000010:1.1:1761231093.333074:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008ac349b2. 00010000:00000010:3.1:1761231093.352270:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:3.1:1761231093.352301:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ec280f6b. 00000080:00000010:1.1:1761231093.411123:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b663cc8f. 00000080:00000010:0.1:1761231093.470889:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000024629fcf. 00010000:00000010:1.1:1761231093.496960:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:1.1:1761231093.496975:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:3.1:1761231093.520878:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e9ebbe2. 00010000:00000010:0.1:1761231093.551090:0:91839:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231093.551097:0:91839:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ec280f6b. 00010000:00000010:3.1:1761231093.594405:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761231093.594414:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:1.1:1761231093.602255:0:91840:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a746fca4. 00010000:00000010:0.1:1761231093.649919:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:0.1:1761231093.649934:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f370345a. 00000080:00000010:0.1:1761231093.655913:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c8ca1bb. 00010000:00000010:0.1:1761231093.699892:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:0.1:1761231093.699900:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:3.1:1761231093.718923:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000701cb90e. 00010000:00000010:3.1:1761231093.752885:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231093.752901:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00000080:00000010:3.1:1761231093.769893:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001f8c49ba. 00010000:00000010:3.1:1761231093.819088:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:3.1:1761231093.819107:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:0.1:1761231093.830308:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ad14d849. 00010000:00000010:0.1:1761231093.870189:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:0.1:1761231093.870203:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005ee4f696. 00000080:00000010:1.1:1761231093.875935:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008ac349b2. 00010000:00000010:0.1:1761231093.914995:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:0.1:1761231093.915017:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00000080:00000010:0.1:1761231093.942899:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f5313e06. 00010000:00000010:1.1:1761231093.989884:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:1.1:1761231093.989891:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00010000:00000010:0.1:1761231094.035371:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:0.1:1761231094.035387:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00010000:00000010:1.1:1761231094.081015:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:1.1:1761231094.081032:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00000080:00000010:3.1:1761231094.115572:0:91849:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b15036c3. 00010000:00000010:3.1:1761231094.153126:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:3.1:1761231094.153142:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:1.1:1761231094.163963:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f03444bf. 00010000:00000010:0.1:1761231094.190159:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:0.1:1761231094.190167:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00010000:00000010:3.1:1761231094.234124:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231094.234131:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:1.1:1761231094.242932:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000afb62743. 00010000:00000010:1.1:1761231094.271892:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:1.1:1761231094.271907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00010000:00000010:1.1:1761231094.308112:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:1.1:1761231094.308129:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00010000:00000010:1.1:1761231094.348695:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761231094.348710:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00010000:00000010:3.1:1761231094.387904:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231094.387920:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:1.1:1761231094.396904:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000022d1833d. 00010000:00000010:1.1:1761231094.434891:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f485d354. 00010000:00000010:1.1:1761231094.434903:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:0.1:1761231094.440502:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003822d66b. 00010000:00000010:3.1:1761231094.482625:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:3.1:1761231094.482632:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000720b7e17. 00010000:00000010:3.1:1761231094.528603:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:3.1:1761231094.528617:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00000080:00000010:1.1:1761231094.539458:0:91859:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000de437705. 00010000:00000010:3.1:1761231094.571356:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:3.1:1761231094.571364:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000abe282bc. 00000080:00000010:1.1:1761231094.580447:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000084e32c56. 00010000:00000010:0.1:1761231094.616164:0:12:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:0.1:1761231094.616184:0:12:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00010000:00000010:1.1:1761231094.660886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:1.1:1761231094.660897:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00010000:00000010:0.1:1761231094.701923:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:0.1:1761231094.701956:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f8fd4482. 00000080:00000010:3.1:1761231094.714100:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b865dd43. 00010000:00000010:1.1:1761231094.748276:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:1.1:1761231094.748292:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000013f3c2e5. 00000080:00000010:1.1:1761231094.758911:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000041ec246a. 00010000:00000010:1.1:1761231094.792882:0:88989:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:1.1:1761231094.792890:0:88989:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00010000:00000010:0.1:1761231094.888899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:0.1:1761231094.888916:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00000080:00000010:0.1:1761231094.912906:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009ba4f828. 00010000:00000010:1.1:1761231094.979919:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761231094.979930:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000013f3c2e5. 00000080:00000010:3.1:1761231094.998996:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000057891e9. 00000080:00000010:3.1:1761231095.037741:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000087fe9bd1. 00010000:00000010:1.1:1761231095.068894:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:1.1:1761231095.068902:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bcb018ff. 00010000:00000010:3.1:1761231095.117051:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:3.1:1761231095.117069:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00010000:00000010:3.1:1761231095.184326:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:3.1:1761231095.184343:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:0.1:1761231095.193331:0:91872:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007b9a2413. 00000080:00000010:0.1:1761231095.253909:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007b9a2413. 00010000:00000010:3.1:1761231095.285009:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:3.1:1761231095.285021:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00000080:00000010:3.1:1761231095.292071:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000000ca4aa8. 00010000:00000010:3.1:1761231095.338209:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:3.1:1761231095.338217:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00010000:00000010:3.1:1761231095.378877:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:3.1:1761231095.378890:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00000080:00000010:1.1:1761231095.392266:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001f8c49ba. 00010000:00000010:3.1:1761231095.419570:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761231095.419582:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000013f3c2e5. 00010000:00000010:1.1:1761231095.471711:0:91877:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:1.1:1761231095.471726:0:91877:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:1.1:1761231095.481078:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c540c55. 00000080:00000010:0.1:1761231095.519081:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000066182a37. 00010000:00000010:0.1:1761231095.550998:0:91879:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:0.1:1761231095.551006:0:91879:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:0.1:1761231095.562360:0:88988:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000029d55bbf. 00010000:00000010:0.1:1761231095.625235:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d067cc47. 00010000:00000010:0.1:1761231095.625244:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:1.1:1761231095.636968:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000b43a2df. 00010000:00000010:0.1:1761231095.671907:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:0.1:1761231095.671914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00000080:00000010:0.1:1761231095.676994:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b15036c3. 00000080:00000010:0.1:1761231095.726924:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f4ae988e. 00010000:00000010:1.1:1761231095.755007:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:1.1:1761231095.755021:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:1.1:1761231095.799890:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000002fba8ac3. 00010000:00000010:1.1:1761231095.829890:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d067cc47. 00010000:00000010:1.1:1761231095.829907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00010000:00000010:1.1:1761231095.862005:0:88989:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761231095.862018:0:88989:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00000080:00000010:0.1:1761231095.867894:0:91888:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b5773b23. 00010000:00000010:1.1:1761231095.900873:0:13:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d067cc47. 00010000:00000010:1.1:1761231095.900885:0:13:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00010000:00000010:0.1:1761231096.017418:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:0.1:1761231096.017430:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00000080:00000010:3.1:1761231096.021128:0:91892:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c610935c. 00010000:00000010:0.1:1761231096.062037:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:0.1:1761231096.062055:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00000080:00000010:3.1:1761231096.076901:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007115da5c. 00010000:00000010:0.1:1761231096.105887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:0.1:1761231096.105901:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000967ce876. 00000080:00000010:1.1:1761231096.121904:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000050033dab. 00010000:00000010:0.1:1761231096.159893:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:0.1:1761231096.159902:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bcb018ff. 00000080:00000010:1.1:1761231096.171884:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000dec5607e. 00010000:00000010:0.1:1761231096.210447:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004807df18. 00010000:00000010:0.1:1761231096.210460:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:1.1:1761231096.219910:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000002194df8c. 00010000:00000010:0.1:1761231096.241889:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:0.1:1761231096.241897:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005170de0b. 00000080:00000010:3.1:1761231096.248884:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000edbeec50. 00010000:00000010:1.1:1761231096.281154:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:1.1:1761231096.281165:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aaebf6f6. 00010000:00000010:3.1:1761231096.324331:0:32:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:3.1:1761231096.324346:0:32:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00010000:00000010:3.1:1761231096.397031:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:3.1:1761231096.397049:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:1.1:1761231096.400991:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b2aa4cab. 00010000:00000010:1.1:1761231096.439960:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000dd6ec541. 00010000:00000010:1.1:1761231096.439970:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ec280f6b. 00000080:00000010:0.1:1761231096.446921:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ddac48d9. 00010000:00000010:1.1:1761231096.478909:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:1.1:1761231096.478925:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00000080:00000010:3.1:1761231096.551128:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000090aa726d. 00010000:00000010:0.1:1761231096.566956:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:0.1:1761231096.566967:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aaebf6f6. 00000080:00000010:1.1:1761231096.573388:0:91904:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b0b500e8. 00010000:00000010:1.1:1761231096.610899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761231096.610923:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:0.1:1761231096.620901:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007b9a2413. 00000080:00000010:1.1:1761231096.710900:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000006f2b404. 00010000:00000010:3.1:1761231096.751926:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:3.1:1761231096.751946:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:1.1:1761231096.769965:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ff74b385. 00010000:00000010:3.1:1761231096.805434:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:3.1:1761231096.805443:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004a1223b6. 00000080:00000010:0.1:1761231096.813516:0:91909:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008c01b32f. 00000080:00000010:0.1:1761231096.860899:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003696a3b3. 00010000:00000010:3.1:1761231096.891911:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:3.1:1761231096.891928:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00010000:00000010:3.1:1761231096.944209:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:3.1:1761231096.944217:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:1.1:1761231096.953015:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fd17af8f. 00010000:00000010:3.1:1761231096.983732:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:3.1:1761231096.983741:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:0.1:1761231096.991317:0:91913:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003696a3b3. 00010000:00000010:3.1:1761231097.017901:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:3.1:1761231097.017922:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00010000:00000010:0.1:1761231097.069964:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:0.1:1761231097.069972:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5207efd. 00000080:00000010:1.1:1761231097.117912:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c2ff289. 00010000:00000010:0.1:1761231097.155906:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:0.1:1761231097.155923:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00010000:00000010:3.1:1761231097.241971:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:3.1:1761231097.242032:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00000080:00000010:1.1:1761231097.256088:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000090aa726d. 00010000:00000010:3.1:1761231097.314896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:3.1:1761231097.314915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00010000:00000010:0.1:1761231097.373086:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:0.1:1761231097.373108:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:0.1:1761231097.431986:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:0.1:1761231097.432007:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00000080:00000010:1.1:1761231097.447154:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008ac349b2. 00010000:00000010:3.1:1761231097.484756:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b033b999. 00010000:00000010:3.1:1761231097.484762:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f1f326dc. 00000080:00000010:0.1:1761231097.500880:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001dfb32da. 00010000:00000010:0.1:1761231097.531901:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:0.1:1761231097.531914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:0.1:1761231097.543134:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f2279b22. 00010000:00000010:3.1:1761231097.584017:0:91925:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:3.1:1761231097.584033:0:91925:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00000080:00000010:1.1:1761231097.589899:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b663cc8f. 00010000:00000010:1.1:1761231097.624891:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:1.1:1761231097.624907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00010000:00000010:0.1:1761231097.679869:0:1033:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:0.1:1761231097.679882:0:1033:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:3.1:1761231097.683884:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000087fe9bd1. 00010000:00000010:0.1:1761231097.716617:0:88989:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:0.1:1761231097.716639:0:88989:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f85cd001. 00000080:00000010:1.1:1761231097.728002:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000024fb2909. 00010000:00000010:1.1:1761231097.761940:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:1.1:1761231097.761961:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aafb0595. 00000080:00000010:1.1:1761231097.772299:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004c6892a7. 00000080:00000010:0.1:1761231097.804896:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000046c1b04c. 00010000:00000010:0.1:1761231097.882968:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:0.1:1761231097.882977:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:1.1:1761231097.891105:0:88988:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001f8c49ba. 00000080:00000010:0.1:1761231097.997882:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f03444bf. 00000080:00000010:1.1:1761231098.036101:0:91935:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d0098e43. 00010000:00000010:3.1:1761231098.074182:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:3.1:1761231098.074190:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000aafb0595. 00000080:00000010:0.1:1761231098.092375:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000093640573. 00010000:00000010:1.1:1761231098.122625:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:1.1:1761231098.122634:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00000080:00000010:0.1:1761231098.209904:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000066182a37. 00010000:00000010:0.1:1761231098.248297:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:0.1:1761231098.248315:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:3.1:1761231098.251939:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000dec5607e. 00010000:00000010:0.1:1761231098.308955:0:91940:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:0.1:1761231098.308969:0:91940:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00000080:00000010:0.1:1761231098.324888:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000014c209f4. 00010000:00000010:0.1:1761231098.352729:0:91940:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:0.1:1761231098.352789:0:91940:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:0.1:1761231098.366729:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e1c1399. 00010000:00000010:0.1:1761231098.413639:0:91942:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:0.1:1761231098.413653:0:91942:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00000080:00000010:3.1:1761231098.419841:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001f8c49ba. 00010000:00000010:1.1:1761231098.453896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:1.1:1761231098.453912:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:0.1:1761231098.461326:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e1c1399. 00010000:00000010:0.1:1761231098.493018:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:0.1:1761231098.493047:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:0.1:1761231098.504971:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000046c1b04c. 00000080:00000010:3.1:1761231098.556983:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009e172779. 00010000:00000010:1.1:1761231098.588988:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761231098.589000:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00000080:00000010:0.1:1761231098.592895:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000926ad9c9. 00010000:00000010:3.1:1761231098.671879:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:3.1:1761231098.671892:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00000080:00000010:0.1:1761231098.694483:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000eb1c2166. 00010000:00000010:3.1:1761231098.717964:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231098.717976:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:3.1:1761231098.734885:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b841249c. 00010000:00000010:3.1:1761231098.757876:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231098.757889:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011645181. 00010000:00000010:3.1:1761231098.795074:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:3.1:1761231098.795095:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f1f326dc. 00000080:00000010:0.1:1761231098.816065:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d159b366. 00010000:00000010:3.1:1761231098.855149:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:3.1:1761231098.855167:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00010000:00000010:1.1:1761231098.915903:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761231098.915920:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00010000:00000010:1.1:1761231098.986903:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:1.1:1761231098.986915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00000080:00000010:3.1:1761231098.996923:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000000ca4aa8. 00010000:00000010:1.1:1761231099.051356:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:1.1:1761231099.051373:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:0.1:1761231099.059962:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000014c209f4. 00010000:00000010:1.1:1761231099.101976:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:1.1:1761231099.101987:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00000080:00000010:1.1:1761231099.115930:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:1.1:1761231099.145895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:1.1:1761231099.145906:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00010000:00000010:0.1:1761231099.186956:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:0.1:1761231099.186977:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00000080:00000010:0.1:1761231099.254257:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cee7568e. 00010000:00000010:1.1:1761231099.279535:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:1.1:1761231099.279553:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00000080:00000010:1.1:1761231099.294040:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000057534bdf. 00000080:00000010:0.1:1761231099.351780:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f5313e06. 00010000:00000010:1.1:1761231099.406392:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:1.1:1761231099.406401:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003eca7fc5. 00010000:00000010:1.1:1761231099.448974:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:1.1:1761231099.448989:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00000080:00000010:3.1:1761231099.459910:0:88989:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000640e53cd. 00010000:00000010:1.1:1761231099.494902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:1.1:1761231099.494915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00010000:00000010:1.1:1761231099.530899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:1.1:1761231099.530918:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:0.1:1761231099.546550:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00010000:00000010:1.1:1761231099.583637:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:1.1:1761231099.583649:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00010000:00000010:1.1:1761231099.623899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:1.1:1761231099.623920:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00000080:00000010:1.1:1761231099.647945:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000e1808a2b. 00010000:00000010:1.1:1761231099.672153:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:1.1:1761231099.672163:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:0.1:1761231099.683943:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000877bbbcd. 00010000:00000010:1.1:1761231099.717892:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:1.1:1761231099.717906:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:0.1:1761231099.728502:0:91969:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000001ff9f0a. 00010000:00000010:1.1:1761231099.806896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:1.1:1761231099.806911:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000810266a2. 00010000:00000010:3.1:1761231099.894159:0:88987:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:3.1:1761231099.894169:0:88987:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00010000:00000010:1.1:1761231099.960653:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:1.1:1761231099.960662:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00000080:00000010:1.1:1761231099.976002:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009e172779. 00010000:00000010:1.1:1761231100.006828:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f485d354. 00010000:00000010:1.1:1761231100.006861:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00000080:00000010:0.1:1761231100.023891:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000002194df8c. 00010000:00000010:1.1:1761231100.084937:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:1.1:1761231100.084952:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f85cd001. 00000080:00000010:0.1:1761231100.086893:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00010000:00000010:1.1:1761231100.111144:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:1.1:1761231100.111156:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00000080:00000010:1.1:1761231100.121694:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000bdc53965. 00010000:00000010:1.1:1761231100.147907:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:1.1:1761231100.147925:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00000080:00000010:1.1:1761231100.186193:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006de9a0c1. 00010000:00000010:1.1:1761231100.205037:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:1.1:1761231100.205053:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00010000:00000010:0.1:1761231100.259896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:0.1:1761231100.259911:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:0.1:1761231100.298025:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:0.1:1761231100.298043:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:3.1:1761231100.318030:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f03444bf. 00010000:00000010:0.1:1761231100.357942:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:0.1:1761231100.357951:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:0.1:1761231100.373889:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007115da5c. 00010000:00000010:0.1:1761231100.401884:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:0.1:1761231100.401907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00000080:00000010:1.1:1761231100.416974:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ad14d849. 00010000:00000010:0.1:1761231100.447911:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:0.1:1761231100.447923:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000d86fc33b. 00010000:00000010:3.1:1761231100.547606:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001c3ee364. 00010000:00000010:3.1:1761231100.547614:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:0.1:1761231100.571900:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000006f2b404. 00010000:00000010:3.1:1761231100.615905:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000032b67042. 00010000:00000010:3.1:1761231100.615925:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5207efd. 00000080:00000010:0.1:1761231100.643948:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ba19c22. 00010000:00000010:1.1:1761231100.663894:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:1.1:1761231100.663910:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00010000:00000010:0.1:1761231147.810520:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d067cc47. 00010000:00000010:0.1:1761231147.810535:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00000080:00000010:0.1:1761231147.831438:0:14:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ff74b385. 00010000:00000010:1.1:1761231147.856256:0:91996:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:1.1:1761231147.856263:0:91996:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004a1223b6. 00000080:00000010:1.1:1761231147.860890:0:91997:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001dfb32da. 00010000:00000010:1.1:1761231147.903419:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:1.1:1761231147.903431:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004a1223b6. 00010000:00000010:1.1:1761231147.943108:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:1.1:1761231147.943125:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00000080:00000010:0.1:1761231147.950024:0:91999:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003278ff9d. 00000080:00000010:1.1:1761231148.044326:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000001ff9f0a. 00010000:00000010:0.1:1761231148.071006:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:0.1:1761231148.071024:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004a1223b6. 00000080:00000010:1.1:1761231148.084892:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000beb91302. 00000080:00000010:1.1:1761231148.141929:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a163506b. 00010000:00000010:3.1:1761231148.169957:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:3.1:1761231148.169972:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005170de0b. 00010000:00000010:3.1:1761231148.235096:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:3.1:1761231148.235112:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00000080:00000010:1.1:1761231148.252119:0:92006:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b6108498. 00010000:00000010:3.1:1761231148.299926:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761231148.299943:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000967ce876. 00010000:00000010:3.1:1761231148.335368:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:3.1:1761231148.335379:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001f876b7f. 00000080:00000010:1.1:1761231148.358596:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000022dc6b9b. 00010000:00000010:3.1:1761231148.391015:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:3.1:1761231148.391028:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000013f3c2e5. 00000080:00000010:1.1:1761231148.400416:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008ac349b2. 00010000:00000010:0.1:1761231148.460566:0:92009:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:0.1:1761231148.460580:0:92009:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:0.1:1761231148.480950:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000eb1c2166. 00010000:00000010:3.1:1761231148.515887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:3.1:1761231148.515902:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000abe282bc. 00000080:00000010:1.1:1761231148.530104:0:20:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000899e612e. 00010000:00000010:0.1:1761231148.576009:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:0.1:1761231148.576024:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000bdb780a0. 00000080:00000010:1.1:1761231148.646820:0:92013:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000046c1b04c. 00010000:00000010:3.1:1761231148.693943:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:3.1:1761231148.693958:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000720b7e17. 00000080:00000010:0.1:1761231148.700885:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000037e0a295. 00000080:00000010:0.1:1761231148.769018:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b5773b23. 00010000:00000010:3.1:1761231148.789895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000342c5436. 00010000:00000010:3.1:1761231148.789906:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231148.803000:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ec25833b. 00010000:00000010:3.1:1761231148.843763:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231148.843777:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00000080:00000010:3.1:1761231148.870953:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004f293f01. 00010000:00000010:3.1:1761231148.915166:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000893a2c7f. 00010000:00000010:3.1:1761231148.915184:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:0.1:1761231148.939944:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a35479a0. 00010000:00000010:3.1:1761231148.974052:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000521ccd61. 00010000:00000010:3.1:1761231148.974075:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:3.1:1761231149.005892:0:88988:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00010000:00000010:0.1:1761231193.963019:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231193.963046:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005ee4f696. 00010000:00000010:3.1:1761231194.042107:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:3.1:1761231194.042132:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00000080:00000010:1.1:1761231194.056887:0:92029:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000014c209f4. 00010000:00000010:0.1:1761231194.153008:0:92029:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000020459bf. 00010000:00000010:0.1:1761231194.153018:0:92029:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f370345a. 00000080:00000010:0.1:1761231194.189917:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006de9a0c1. 00010000:00000010:0.1:1761231194.218908:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:0.1:1761231194.218932:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00010000:00000010:1.1:1761231194.291901:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000032b67042. 00010000:00000010:1.1:1761231194.291921:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005b9c88b6. 00000080:00000010:0.1:1761231194.316594:0:92032:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007b1d315f. 00010000:00000010:0.1:1761231194.386034:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000001c3ee364. 00010000:00000010:0.1:1761231194.386053:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:0.1:1761231194.410902:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000daad9a21. 00010000:00000010:3.1:1761231194.483916:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:3.1:1761231194.483940:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00010000:00000010:1.1:1761231194.582117:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:1.1:1761231194.582128:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005b9c88b6. 00000080:00000010:1.1:1761231194.601912:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000647ca513. 00000080:00000010:1.1:1761231194.668842:0:19:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000daad9a21. 00010000:00000010:1.1:1761231194.744925:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:1.1:1761231194.744949:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:3.1:1761231194.762089:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000b43a2df. 00010000:00000010:0.1:1761231194.836306:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:0.1:1761231194.836328:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005b9c88b6. 00010000:00000010:3.1:1761231194.928948:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:3.1:1761231194.928955:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:1.1:1761231194.997883:0:88989:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ad14d849. 00010000:00000010:3.1:1761231195.029946:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231195.029970:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00010000:00000010:1.1:1761231195.091575:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:1.1:1761231195.091602:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00000080:00000010:0.1:1761231195.120566:0:88987:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000926ad9c9. 00010000:00000010:1.1:1761231195.188213:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231195.188227:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ad7a691b. 00000080:00000010:1.1:1761231195.206777:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000926e51ff. 00010000:00000010:3.1:1761231195.251082:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:3.1:1761231195.251096:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:1.1:1761231195.278981:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b663cc8f. 00010000:00000010:3.1:1761231195.323423:0:92044:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231195.323442:0:92044:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f85cd001. 00000080:00000010:3.1:1761231195.330907:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000bd456729. 00000080:00000010:1.1:1761231195.401107:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001f8c49ba. 00010000:00000010:3.1:1761231195.450026:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000004a10698. 00010000:00000010:3.1:1761231195.450043:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f85cd001. 00010000:00000010:0.1:1761231195.502430:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:0.1:1761231195.502451:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00010000:00000010:1.1:1761231195.570226:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231195.570239:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00000080:00000010:3.1:1761231195.571507:0:92049:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000037a17b0d. 00000080:00000010:3.1:1761231195.629010:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00010000:00000010:1.1:1761231195.741196:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:1.1:1761231195.741207:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00010000:00000010:1.1:1761231195.803508:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:1.1:1761231195.803524:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ad7a691b. 00000080:00000010:0.1:1761231195.828077:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000018f9a6ac. 00000080:00000010:1.1:1761231195.872161:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000081008c10. 00010000:00000010:3.1:1761231195.921366:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:3.1:1761231195.921373:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00000080:00000010:0.1:1761231195.941921:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000000ca4aa8. 00010000:00000010:1.1:1761231195.977189:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:1.1:1761231195.977206:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00000080:00000010:0.1:1761231195.997922:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f03444bf. 00010000:00000010:3.1:1761231196.036023:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:3.1:1761231196.036040:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:0.1:1761231196.057909:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fd17af8f. 00000080:00000010:1.1:1761231196.139085:0:92058:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000000c540c55. 00010000:00000010:3.1:1761231196.201109:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000006d18d21b. 00010000:00000010:3.1:1761231196.201126:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00000080:00000010:0.1:1761231196.233889:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fd17af8f. 00010000:00000010:3.1:1761231196.262362:0:92059:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000ebad5453. 00010000:00000010:3.1:1761231196.262370:0:92059:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00000080:00000010:3.1:1761231196.277907:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000055b04bfa. 00010000:00000010:0.1:1761231196.301891:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:0.1:1761231196.301909:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c4969726. 00010000:00000010:3.1:1761231196.355786:0:32:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:3.1:1761231196.355792:0:32:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00010000:00000010:3.1:1761231196.393884:0:32:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231196.393891:0:32:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00000080:00000010:0.1:1761231196.403024:0:88988:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000eb1c2166. 00010000:00000010:1.1:1761231196.439759:0:13:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761231196.439772:0:13:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:3.1:1761231196.444898:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ec6b65ac. 00010000:00000010:0.1:1761231196.479003:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:0.1:1761231196.479019:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:3.1:1761231196.520303:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:3.1:1761231196.520328:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:1.1:1761231196.533917:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000026945b88. 00010000:00000010:1.1:1761231196.570880:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000028b11d35. 00010000:00000010:1.1:1761231196.570892:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:0.1:1761231196.583909:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000018ca2c0b. 00010000:00000010:3.1:1761231196.661768:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:3.1:1761231196.661786:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c930e17. 00000080:00000010:1.1:1761231196.664027:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000079b4f4f9. 00010000:00000010:3.1:1761231196.762996:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:3.1:1761231196.763013:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00000080:00000010:1.1:1761231196.794940:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000087fe9bd1. 00010000:00000010:1.1:1761231196.825894:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:1.1:1761231196.825913:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00010000:00000010:3.1:1761231196.909961:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:3.1:1761231196.909981:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000db46578c. 00010000:00000010:0.1:1761231196.957363:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:0.1:1761231196.957370:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00000080:00000010:0.1:1761231196.970883:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f5313e06. 00010000:00000010:0.1:1761231197.002884:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:0.1:1761231197.002892:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00010000:00000010:3.1:1761231197.050370:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:3.1:1761231197.050379:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00010000:00000010:0.1:1761231197.088913:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:0.1:1761231197.088927:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00000080:00000010:0.1:1761231197.103710:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000de437705. 00010000:00000010:3.1:1761231197.143937:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:3.1:1761231197.143952:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000482f0ed0. 00010000:00000010:3.1:1761231197.195379:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:3.1:1761231197.195392:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00010000:00000010:3.1:1761231197.232882:0:92079:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:3.1:1761231197.232896:0:92079:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00000080:00000010:1.1:1761231197.239020:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b2cb5a1d. 00000080:00000010:0.1:1761231197.282918:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:3.1:1761231197.309990:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:3.1:1761231197.310003:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00010000:00000010:3.1:1761231197.361903:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c682eeae. 00010000:00000010:3.1:1761231197.361917:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:0.1:1761231197.365917:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000dec5607e. 00010000:00000010:0.1:1761231197.395877:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:0.1:1761231197.395884:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00000080:00000010:3.1:1761231197.413909:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b5773b23. 00010000:00000010:3.1:1761231197.441942:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:3.1:1761231197.441960:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c589180f. 00000080:00000010:1.1:1761231197.473891:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000029324c07. 00010000:00000010:0.1:1761231197.500787:0:1033:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:0.1:1761231197.500798:0:1033:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:0.1:1761231197.506369:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000daad9a21. 00010000:00000010:0.1:1761231197.544888:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:0.1:1761231197.544897:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00000080:00000010:3.1:1761231197.557486:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000018ca2c0b. 00010000:00000010:1.1:1761231197.596306:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:1.1:1761231197.596315:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00010000:00000010:0.1:1761231197.636879:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:0.1:1761231197.636887:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:1.1:1761231197.647282:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005ecf7799. 00010000:00000010:0.1:1761231197.669017:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:0.1:1761231197.669032:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00000080:00000010:0.1:1761231197.688895:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000930c75c8. 00010000:00000010:0.1:1761231197.728889:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:0.1:1761231197.728903:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00000080:00000010:3.1:1761231197.745988:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006ae998d0. 00010000:00000010:3.1:1761231197.802885:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:3.1:1761231197.802894:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00000080:00000010:1.1:1761231197.818897:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000926ad9c9. 00010000:00000010:0.1:1761231197.862094:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:0.1:1761231197.862112:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00010000:00000010:0.1:1761231197.906978:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:0.1:1761231197.907006:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:3.1:1761231197.977059:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:3.1:1761231197.977075:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:0.1:1761231197.990979:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003f0d8b90. 00010000:00000010:1.1:1761231198.035013:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:1.1:1761231198.035042:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c930e17. 00000080:00000010:0.1:1761231198.057367:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003ace7c30. 00010000:00000010:1.1:1761231198.088902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761231198.088916:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000007c2fd66c. 00010000:00000010:1.1:1761231198.124908:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:1.1:1761231198.124926:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00010000:00000010:1.1:1761231198.176910:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761231198.176926:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fbf7778a. 00000080:00000010:0.1:1761231198.189002:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000024629fcf. 00010000:00000010:3.1:1761231198.237183:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:3.1:1761231198.237189:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001334057e. 00000080:00000010:3.1:1761231198.244095:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f2279b22. 00000080:00000010:0.1:1761231198.282068:0:92101:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004c6892a7. 00000080:00000010:1.1:1761231198.335892:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003290d8cb. 00010000:00000010:0.1:1761231198.364251:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:0.1:1761231198.364268:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00010000:00000010:0.1:1761231198.415211:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:0.1:1761231198.415226:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00000080:00000010:0.1:1761231198.421893:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000000ca4aa8. 00010000:00000010:0.1:1761231198.457658:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:0.1:1761231198.457675:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b85a4a73. 00000080:00000010:1.1:1761231198.482386:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c9357289. 00010000:00000010:0.1:1761231198.539883:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:0.1:1761231198.539893:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c4969726. 00010000:00000010:3.1:1761231198.588877:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231198.588890:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f2aa9f4c. 00000080:00000010:3.1:1761231198.592891:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000044aec3ee. 00010000:00000010:3.1:1761231198.616878:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007ce65a47. 00010000:00000010:3.1:1761231198.616898:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000c1650f5f. 00000080:00000010:0.1:1761231198.621593:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000451f8ae4. 00010000:00000010:0.1:1761231198.649893:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b033b999. 00010000:00000010:0.1:1761231198.649911:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000ad7a691b. 00000080:00000010:1.1:1761231198.666877:0:92109:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000592169e1. 00010000:00000010:0.1:1761231198.709205:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000f53066fa. 00010000:00000010:0.1:1761231198.709216:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a5b2690c. 00000080:00000010:3.1:1761231198.718289:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000592169e1. 00010000:00000010:0.1:1761231198.759749:0:1356:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:0.1:1761231198.759766:0:1356:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005b9c88b6. 00010000:00000010:1.1:1761231198.818952:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:1.1:1761231198.818973:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00000080:00000010:0.1:1761231198.832129:0:92112:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c610935c. 00010000:00000010:0.1:1761231198.867994:0:92113:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:0.1:1761231198.868006:0:92113:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:0.1:1761231198.876134:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000787c0332. 00010000:00000010:0.1:1761231198.911886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c2bbf93d. 00010000:00000010:0.1:1761231198.911907:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00010000:00000010:1.1:1761231198.957886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:1.1:1761231198.957895:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:3.1:1761231198.987384:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009ba4f828. 00010000:00000010:1.1:1761231199.010912:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000c622aac7. 00010000:00000010:1.1:1761231199.010920:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f370345a. 00000080:00000010:0.1:1761231199.025988:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006fc51f99. 00000080:00000010:0.1:1761231199.068890:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005e438bbf. 00010000:00000010:0.1:1761231199.100884:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231199.100900:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00010000:00000010:3.1:1761231199.148889:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231199.148906:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00000080:00000010:0.1:1761231199.166187:0:92119:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004f293f01. 00010000:00000010:0.1:1761231199.201896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009bcc0a1a. 00010000:00000010:0.1:1761231199.201932:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001a94e9ca. 00000080:00000010:1.1:1761231199.211397:0:92120:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000006e9ebbe2. 00010000:00000010:3.1:1761231199.248928:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000027bc25de. 00010000:00000010:3.1:1761231199.248938:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f370345a. 00000080:00000010:3.1:1761231199.259931:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a20efd9d. 00010000:00000010:3.1:1761231199.345887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b1268b24. 00010000:00000010:3.1:1761231199.345902:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000005ee4f696. 00010000:00000010:3.1:1761231199.391905:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:3.1:1761231199.391922:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00000080:00000010:0.1:1761231199.411442:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c4a6764b. 00010000:00000010:3.1:1761231199.518023:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231199.518037:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00000080:00000010:1.1:1761231199.535898:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ac373ff. 00010000:00000010:3.1:1761231199.604898:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231199.604914:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00010000:00000010:0.1:1761231199.662910:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a920ad04. 00010000:00000010:0.1:1761231199.662925:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000048b081f2. 00000080:00000010:0.1:1761231199.678008:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001174e0d2. 00010000:00000010:3.1:1761231199.719892:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000034481ee6. 00010000:00000010:3.1:1761231199.719908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f15a1199. 00000080:00000010:0.1:1761231199.744887:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f35f02da. 00010000:00000010:3.1:1761231199.764681:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007598d4c3. 00010000:00000010:3.1:1761231199.764697:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000077b3fb8a. 00000080:00000010:1.1:1761231199.781124:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ff74b385. 00010000:00000010:1.1:1761231199.817901:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231199.817920:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00000080:00000010:3.1:1761231199.825339:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003f0d8b90. 00010000:00000010:0.1:1761231199.858205:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:0.1:1761231199.858214:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000006417d575. 00000080:00000010:1.1:1761231199.862029:0:92132:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fd17af8f. 00010000:00000010:0.1:1761231199.901053:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a220292. 00010000:00000010:0.1:1761231199.901074:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000720b7e17. 00000080:00000010:0.1:1761231199.913880:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000081008c10. 00010000:00000010:0.1:1761231199.970879:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a312ec3b. 00010000:00000010:0.1:1761231199.970892:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00000080:00000010:0.1:1761231199.970909:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b15036c3. 00000080:00000010:1.1:1761231200.029631:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000018f9a6ac. 00000080:00000010:1.1:1761231200.133886:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b663cc8f. 00010000:00000010:3.1:1761231200.166509:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:3.1:1761231200.166526:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00000080:00000010:0.1:1761231246.053880:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000cd8b3a27. 00010000:00000010:0.1:1761231246.074900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:0.1:1761231246.074910:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:0.1:1761231246.111896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:0.1:1761231246.111905:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00010000:00000010:3.1:1761231246.126874:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231246.126880:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231246.129908:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:0.1:1761231246.143884:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231246.143891:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00000080:00000010:3.1:1761231246.147914:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b2aa4cab. 00010000:00000010:3.1:1761231246.157867:0:92152:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231246.157872:0:92152:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231246.157876:0:92152:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ddac48d9. 00010000:00000010:3.1:1761231246.170879:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231246.170883:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:3.1:1761231246.174869:0:92153:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000dec5607e. 00010000:00000010:3.1:1761231246.187873:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231246.187880:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231246.187884:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:3.1:1761231246.205870:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231246.205878:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:3.1:1761231246.205893:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f03444bf. 00010000:00000010:3.1:1761231246.220890:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:3.1:1761231246.220898:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231246.220902:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ddac48d9. 00010000:00000010:1.1:1761231246.226869:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:1.1:1761231246.226875:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00010000:00000010:3.1:1761231246.255879:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:3.1:1761231246.255886:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00010000:00000010:3.1:1761231246.290893:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:3.1:1761231246.290903:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00010000:00000010:3.1:1761231246.320883:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:3.1:1761231246.320895:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00010000:00000010:3.1:1761231246.346893:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:3.1:1761231246.346905:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00000080:00000010:1.1:1761231246.388877:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b2aa4cab. 00000080:00000010:3.1:1761231246.422922:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a7dd8716. 00000080:00000010:0.1:1761231246.442889:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001444fa72. 00000080:00000010:1.1:1761231246.453901:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000600a1cad. 00000080:00000010:3.1:1761231246.472884:0:92173:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b2aa4cab. 00000080:00000010:3.1:1761231246.488890:0:92174:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000020d6648a. 00000080:00000010:3.1:1761231246.510888:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ddac48d9. 00000080:00000010:3.1:1761231246.534897:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000020d6648a. 00000080:00000010:0.1:1761231246.540918:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003f0d8b90. 00010000:00000010:3.1:1761231246.560920:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231246.560932:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231246.560945:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ddac48d9. 00010000:00000010:3.1:1761231246.595867:0:92180:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231246.595874:0:92180:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00000080:00000010:3.1:1761231246.603866:0:92180:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fd1cf851. 00010000:00000010:1.1:1761231246.617893:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:1.1:1761231246.617900:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:1.1:1761231246.622882:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004342541f. 00000080:00000010:0.1:1761231246.642889:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00000080:00000010:1.1:1761231246.648881:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d7db8d9f. 00000080:00000010:0.1:1761231246.662879:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009ab9ba8c. 00010000:00000010:0.1:1761231246.678875:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:0.1:1761231246.678882:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:1.1:1761231246.678887:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f03444bf. 00000080:00000010:0.1:1761231246.691898:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00000080:00000010:1.1:1761231246.768881:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000037a17b0d. 00000080:00000010:0.1:1761231246.939890:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000087fe9bd1. 00000080:00000010:0.1:1761231246.959919:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ddac48d9. 00000080:00000010:3.1:1761231246.978919:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003f0d8b90. 00000080:00000010:3.1:1761231246.998938:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00000080:00000010:3.1:1761231247.023890:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000600a1cad. 00000080:00000010:3.1:1761231247.044899:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00000080:00000010:3.1:1761231247.062860:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000600a1cad. 00010000:00000010:1.1:1761231247.122868:0:88988:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:1.1:1761231247.122875:0:88988:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00010000:00000010:1.1:1761231247.147869:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:1.1:1761231247.147876:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00010000:00000010:1.1:1761231247.169883:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:1.1:1761231247.169890:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00010000:00000010:1.1:1761231247.200902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:1.1:1761231247.200909:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:1.1:1761231247.229886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:1.1:1761231247.229894:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00010000:00000010:1.1:1761231247.265900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761231247.265912:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:0.1:1761231247.290920:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004f293f01. 00010000:00000010:1.1:1761231247.307900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:1.1:1761231247.307911:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00000080:00000010:3.1:1761231247.329872:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b2aa4cab. 00000080:00000010:3.1:1761231247.350877:0:92225:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00000080:00000010:3.1:1761231247.363917:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003f0d8b90. 00000080:00000010:1.1:1761231247.378899:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fa532857. 00000080:00000010:3.1:1761231247.457873:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000600a1cad. 00000080:00000010:3.1:1761231247.470874:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005df662d6. 00000080:00000010:3.1:1761231247.485924:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00000080:00000010:3.1:1761231247.498908:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000600a1cad. 00000080:00000010:3.1:1761231247.511881:0:92236:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000a7dd8716. 00000080:00000010:3.1:1761231247.527880:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000003f0d8b90. 00000080:00000010:0.1:1761231247.556895:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001444fa72. 00000080:00000010:3.1:1761231247.665895:0:92244:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00000080:00000010:3.1:1761231247.679931:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005df662d6. 00000080:00000010:3.1:1761231247.696873:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:3.1:1761231247.709883:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:3.1:1761231247.709889:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:3.1:1761231247.713889:0:92247:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001444fa72. 00000080:00000010:3.1:1761231247.727888:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:0.1:1761231247.744905:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:0.1:1761231247.744938:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:3.1:1761231247.748865:0:92249:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000600a1cad. 00000080:00000010:3.1:1761231247.765912:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:0.1:1761231247.774873:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:0.1:1761231247.774878:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00000080:00000010:3.1:1761231247.779870:0:92251:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000600a1cad. 00000080:00000010:3.1:1761231247.791866:0:92252:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001444fa72. 00010000:00000010:0.1:1761231247.804883:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231247.804891:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:3.1:1761231247.808862:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00000080:00000010:3.1:1761231247.822864:0:92254:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005df662d6. 00010000:00000010:0.1:1761231247.833911:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:0.1:1761231247.833918:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231247.838868:0:92255:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00000080:00000010:3.1:1761231247.852866:0:92256:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000b2aa4cab. 00010000:00000010:0.1:1761231247.864881:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:0.1:1761231247.864886:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231247.868875:0:92257:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005df662d6. 00000080:00000010:3.1:1761231247.881886:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:0.1:1761231247.887892:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:0.1:1761231247.887900:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00000080:00000010:3.1:1761231247.906907:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001444fa72. 00010000:00000010:0.1:1761231247.923913:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00000080:00000010:3.1:1761231247.923917:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005df662d6. 00010000:00000010:0.1:1761231247.923921:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:3.1:1761231247.940931:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:0.1:1761231247.958881:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:0.1:1761231247.958889:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00000080:00000010:3.1:1761231247.962862:0:92263:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004342541f. 00000080:00000010:3.1:1761231247.982872:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:0.1:1761231247.989909:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:0.1:1761231247.989917:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:0.1:1761231247.992905:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000296aace5. 00010000:00000010:0.1:1761231248.022908:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:0.1:1761231248.022917:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00010000:00000010:0.1:1761231248.054891:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:0.1:1761231248.054899:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00000080:00000010:3.1:1761231248.080905:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:0.1:1761231248.088899:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:0.1:1761231248.088908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:0.1:1761231248.091900:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ddac48d9. 00010000:00000010:0.1:1761231248.126887:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:0.1:1761231248.126897:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00010000:00000010:0.1:1761231248.165943:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:0.1:1761231248.165953:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00000080:00000010:3.1:1761231248.169883:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001444fa72. 00000080:00000010:3.1:1761231248.194867:0:92277:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000020d6648a. 00010000:00000010:0.1:1761231248.206886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:0.1:1761231248.206895:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231248.214938:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001444fa72. 00000080:00000010:1.1:1761231248.232891:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005df662d6. 00010000:00000010:0.1:1761231248.256889:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000007021ecbc. 00010000:00000010:0.1:1761231248.256909:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00000080:00000010:0.1:1761231248.261907:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005ecf7799. 00010000:00000010:0.1:1761231248.300980:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:0.1:1761231248.300991:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00010000:00000010:0.1:1761231248.354915:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:0.1:1761231248.354930:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00010000:00000010:0.1:1761231248.406912:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:0.1:1761231248.406928:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00010000:00000010:0.1:1761231248.459911:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:0.1:1761231248.459920:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:0.1:1761231248.473919:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:0.1:1761231248.473927:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00000080:00000010:3.1:1761231248.505916:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001444fa72. 00010000:00000010:1.1:1761231248.528939:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00010000:00000010:1.1:1761231248.528954:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:3.1:1761231248.532911:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00010000:00000010:0.1:1761231248.579929:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000b547bc24. 00010000:00000010:0.1:1761231248.579945:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00010000:00000010:0.1:1761231248.626910:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:0.1:1761231248.626922:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000dc50a3dd. 00010000:00000010:0.1:1761231248.670924:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:0.1:1761231248.670936:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001af1687a. 00000080:00000010:3.1:1761231248.702952:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000000ca4aa8. 00010000:00000010:0.1:1761231248.721904:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:0.1:1761231248.721915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00000080:00000010:3.1:1761231248.725927:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000d7db8d9f. 00000080:00000010:3.1:1761231248.750905:0:92300:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c4a6764b. 00010000:00000010:0.1:1761231248.775922:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231248.775935:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:3.1:1761231248.779910:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c4a6764b. 00010000:00000010:3.1:1761231248.798915:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231248.798924:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:3.1:1761231248.803919:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ac373ff. 00010000:00000010:0.1:1761231248.824908:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:0.1:1761231248.824922:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000090a855e4. 00010000:00000010:3.1:1761231248.850910:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:3.1:1761231248.850934:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000f29a8cff. 00010000:00000010:0.1:1761231248.877895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000820aadac. 00010000:00000010:0.1:1761231248.877908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003df2529a. 00010000:00000010:3.1:1761231248.901880:0:92306:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231248.901890:0:92306:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:3.1:1761231248.906903:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:0.1:1761231248.925928:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000cc505d99. 00010000:00000010:0.1:1761231248.925943:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00010000:00000010:3.1:1761231248.950931:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e67b7f01. 00010000:00000010:3.1:1761231248.950948:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000145af001. 00010000:00000010:0.1:1761231248.978928:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009cdc52ee. 00010000:00000010:0.1:1761231248.978941:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000071ec7642. 00010000:00000010:3.1:1761231249.003927:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000005fec6e1b. 00010000:00000010:3.1:1761231249.003941:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e2e2895a. 00010000:00000010:0.1:1761231249.029920:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000057ce6cd1. 00010000:00000010:0.1:1761231249.029931:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000008942a157. 00010000:00000010:3.1:1761231249.054893:0:92311:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231249.054901:0:92311:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000093b0a806. 00000080:00000010:3.1:1761231249.059900:0:92312:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005ecf7799. 00010000:00000010:0.1:1761231249.083902:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:0.1:1761231249.083915:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:3.1:1761231249.087923:0:92313:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:0.1:1761231249.141922:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004fb9c30f. 00010000:00000010:0.1:1761231249.141936:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000004e97244f. 00010000:00000010:3.1:1761231249.165895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000e3a6d8f6. 00010000:00000010:3.1:1761231249.165908:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000037031d06. 00010000:00000010:0.1:1761231249.193905:0:88670:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:0.1:1761231249.193919:0:88670:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b71003a4. 00000080:00000010:1.1:1761231249.199907:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 0000000000ca4aa8. 00010000:00000010:3.1:1761231249.219919:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231249.219927:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00000080:00000010:3.1:1761231249.228979:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009ab9ba8c. 00010000:00000010:0.1:1761231249.244900:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:0.1:1761231249.244924:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00000080:00000010:3.1:1761231249.248873:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001ac373ff. 00010000:00000010:3.1:1761231249.265905:0:92320:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000872c0a66. 00010000:00000010:3.1:1761231249.265917:0:92320:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001aa83afc. 00010000:00000010:1.1:1761231249.286889:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:1.1:1761231249.286896:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000001d12a12f. 00000080:00000010:3.1:1761231249.291882:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000ec25833b. 00010000:00000010:0.1:1761231249.308884:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000059331689. 00010000:00000010:0.1:1761231249.308892:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000016fa36d4. 00010000:00000010:1.1:1761231249.321886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000db43910f. 00010000:00000010:1.1:1761231249.321894:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231249.325869:0:92323:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007115da5c. 00010000:00000010:0.1:1761231249.351873:0:88995:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000d03a1fb1. 00000080:00000010:3.1:1761231249.351875:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c4a6764b. 00010000:00000010:0.1:1761231249.351880:0:88995:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:3.1:1761231249.364867:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:0.1:1761231249.374928:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009082912f. 00010000:00000010:0.1:1761231249.374942:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000011025539. 00000080:00000010:0.1:1761231249.378896:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000fd1cf851. 00010000:00000010:1.1:1761231249.388872:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000585c4323. 00010000:00000010:1.1:1761231249.388877:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000002fc2ed66. 00010000:00000010:0.1:1761231249.401871:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000a884655f. 00010000:00000010:0.1:1761231249.401877:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 000000003d8f60ce. 00010000:00000010:1.1:1761231249.418895:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 00000000617a6315. 00010000:00000010:1.1:1761231249.418905:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000634e9154. 00010000:00000010:0.1:1761231249.425890:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000009a4f98d5. 00010000:00000010:0.1:1761231249.425896:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:3.1:1761231249.433863:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:1.1:1761231249.443881:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:1.1:1761231249.443886:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000b94e9b1c. 00000080:00000010:3.1:1761231249.446866:0:92331:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000007115da5c. 00010000:00000010:0.1:1761231249.455888:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:0.1:1761231249.455897:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000fc168aad. 00000080:00000010:3.1:1761231249.459886:0:92332:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000384262c1. 00010000:00000010:3.1:1761231249.478863:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231249.478869:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:3.1:1761231249.482892:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f03444bf. 00010000:00000010:3.1:1761231249.487876:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231249.487882:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:3.1:1761231249.491877:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c4a6764b. 00010000:00000010:3.1:1761231249.499909:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00000080:00000010:1.1:1761231249.499911:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000001f8c49ba. 00010000:00000010:3.1:1761231249.499917:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00010000:00000010:3.1:1761231249.511889:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231249.511896:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000e9fae5c0. 00000080:00000010:0.1:1761231249.515871:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000f03444bf. 00010000:00000010:3.1:1761231249.524860:0:92337:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 000000004aa7ca7e. 00010000:00000010:3.1:1761231249.524866:0:92337:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 00000000a209a555. 00000080:00000010:1.1:1761231249.529878:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000005e438bbf. 00000080:00000010:0.1:1761231249.541894:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000008404881e. 00010000:00000010:3.1:1761231249.557896:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231249.557903:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:3.1:1761231249.561872:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009b209f02. 00010000:00000010:3.1:1761231249.575890:0:90550:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231249.575894:0:90550:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231249.575901:0:90550:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004342541f. 00010000:00000010:3.1:1761231249.585871:0:92341:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231249.585877:0:92341:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:3.1:1761231249.590877:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c4a6764b. 00010000:00000010:3.1:1761231249.595885:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231249.595889:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231249.603908:0:90550:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000004342541f. 00010000:00000010:3.1:1761231249.613865:0:92343:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000051f26aa3. 00010000:00000010:3.1:1761231249.613870:0:92343:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000075e0ad4b. 00000080:00000010:3.1:1761231249.617861:0:90549:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 00000000c4a6764b. 00010000:00000010:3.1:1761231249.631886:0:0:0:(ldlm_resource.c:1463:__ldlm_resource_free()) slab-freed '(res)': 224 at 0000000079ac4d98. 00010000:00000010:3.1:1761231249.631892:0:0:0:(ldlm_lock.c:197:lock_handle_free()) slab-freed 'lock': 560 at 0000000029507fad. 00000080:00000010:3.1:1761231249.639883:0:0:0:(dcache.c:49:free_dentry_data()) kfreed 'lld': 24 at 000000009b209f02. 00000800:00000200:0.1:1761231269.581534:0:88997:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761231275.725673:0:88994:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761231280.845270:0:88995:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761231285.965323:0:88996:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761231292.749967:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761231292.750040:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761231304.013670:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761231308.109639:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761231308.109714:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761231320.398176:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761231324.494247:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761231324.494378:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231336.653984:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231336.653986:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 1, 1 secs later) 00000800:00000200:0.1:1761231336.654208:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231336.654210:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 1, 1 secs later) 00000800:00000200:0.1:1761231336.654353:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231336.654355:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 1, 1 secs later) 00000800:00000200:0.1:1761231341.773485:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231341.773651:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231341.773750:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231403.213764:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231403.213768:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 20 secs later) 00000800:00000200:0.1:1761231403.214165:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231403.214168:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 20 secs later) 00000800:00000200:0.1:1761231403.214524:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231403.214526:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 20 secs later) 00000800:00000200:0.1:1761231408.333815:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231408.333818:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 15 secs later) 00000800:00000200:0.1:1761231408.334278:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231408.334281:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 15 secs later) 00000800:00000200:0.1:1761231408.334614:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231408.334618:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 15 secs later) 00000800:00000200:0.1:1761231413.453782:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231413.453790:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 10 secs later) 00000800:00000200:0.1:1761231413.454323:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231413.454325:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 10 secs later) 00000800:00000200:0.1:1761231413.454584:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231413.454586:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 10 secs later) 00000800:00000200:0.1:1761231418.573730:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231418.573733:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 5 secs later) 00000800:00000200:0.1:1761231418.574186:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231418.574189:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 5 secs later) 00000800:00000200:0.1:1761231418.574472:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231418.574474:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 32, 5 secs later) 00000800:00000200:0.1:1761231423.693901:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231423.694097:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231423.694290:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231428.814103:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231428.814108:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231428.814716:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231428.814719:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231428.815222:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231428.815225:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231433.933580:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231433.933585:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1761231433.934104:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231433.934107:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1761231433.934364:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231433.934367:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1761231439.053834:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231439.053838:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761231439.054252:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231439.054255:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761231439.054504:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231439.054507:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761231449.293839:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231449.293877:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761231449.294396:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231449.294399:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761231449.294669:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231449.294671:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761231454.413780:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231454.413785:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231454.414208:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231454.414210:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231454.414505:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231454.414508:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231459.534034:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231459.534038:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231459.534460:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231459.534462:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231459.534719:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231459.534721:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231464.653721:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231464.653726:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231464.654232:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231464.654235:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231464.654495:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231464.654498:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231469.773823:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231469.773827:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231469.774335:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231469.774338:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231469.774600:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231469.774603:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231474.893800:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231474.893804:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1761231474.894282:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231474.894284:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1761231474.894539:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231474.894541:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1761231480.014163:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231480.014167:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1761231480.014544:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231480.014547:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1761231480.014797:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231480.014799:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1761231485.133927:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231485.133930:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761231485.134244:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231485.134247:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761231485.134589:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231485.134591:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761231490.253839:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231490.254122:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231490.254325:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231495.373820:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231495.373824:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231495.374503:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231495.374507:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231495.375030:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231495.375034:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231500.494078:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231500.494082:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231500.494424:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231500.494426:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231500.494842:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231500.494874:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231505.614154:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231505.614159:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761231505.614559:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231505.614562:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761231505.614821:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231505.614824:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761231510.733789:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231510.733793:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761231510.734218:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231510.734220:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761231510.734497:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231510.734499:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761231515.853454:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231515.853458:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761231515.853714:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231515.853716:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761231515.853873:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231515.853874:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761231520.974172:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231520.974176:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231520.974627:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231520.974629:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231520.974938:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231520.974941:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231526.093443:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231526.093445:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231526.093610:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231526.093611:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231526.093788:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231526.093789:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231531.213383:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231531.213389:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231531.213644:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231531.213645:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231531.213779:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231531.213780:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231567.054123:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231567.054128:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231567.054511:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231567.054514:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231567.054797:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231567.054800:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231572.174057:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231572.174065:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761231572.174595:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231572.174597:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761231572.174902:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231572.174904:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761231577.294002:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231577.294008:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761231577.294410:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231577.294413:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761231577.294782:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231577.294785:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761231582.413896:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231582.413900:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761231582.414239:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231582.414241:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761231582.414692:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231582.414695:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761231587.533458:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231587.533460:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1761231587.533701:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231587.533703:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1761231587.533878:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231587.533879:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1761231592.653937:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231592.653942:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1761231592.654380:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231592.654382:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1761231592.654622:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231592.654624:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1761231597.774030:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231597.774033:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1761231597.774401:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231597.774404:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1761231597.774761:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231597.774763:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1761231602.893526:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231602.893530:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231602.893926:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231602.893928:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231602.894211:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231602.894213:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231608.013778:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231608.013781:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231608.014234:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231608.014237:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231608.014593:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231608.014596:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231613.133584:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231613.133589:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231613.133809:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231613.133810:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231613.134021:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231613.134022:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231618.253448:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231618.253450:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231618.253622:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231618.253623:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231618.253807:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231618.253808:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231654.093596:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231654.093599:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231654.093877:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231654.093879:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231654.094044:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231654.094045:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231659.213692:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231659.213699:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231659.214057:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231659.214059:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231659.214192:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231659.214193:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231669.453727:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231669.453730:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231669.454132:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231669.454134:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231669.454398:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231669.454400:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231674.573952:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231674.573956:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231674.574434:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231674.574437:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231674.574686:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231674.574688:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231679.693540:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231679.693542:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231679.693719:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231679.693720:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231679.694076:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231679.694077:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231684.813420:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231684.813423:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231684.813630:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231684.813631:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231684.813757:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231684.813758:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231689.934092:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231689.934364:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231689.934579:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231695.053796:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231695.053800:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231695.054249:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231695.054253:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231695.054772:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231695.054775:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231746.253838:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231746.253879:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231746.254327:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231746.254330:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231746.254653:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231746.254655:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231751.374209:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231751.374218:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231751.374840:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231751.374902:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231751.375207:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231751.375210:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231756.493971:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231756.494206:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231756.494389:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231761.614252:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231761.614259:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231761.614986:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231761.614991:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231761.615443:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231761.615446:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231766.733954:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231766.733958:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231766.734358:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231766.734360:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231766.734637:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231766.734639:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231771.853824:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231771.853828:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761231771.854263:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231771.854266:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761231771.854523:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231771.854526:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761231776.973512:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231776.973514:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1761231776.973774:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231776.973776:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1761231776.973981:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231776.973983:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1761231782.093966:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231782.093970:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761231782.094300:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231782.094302:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761231782.094692:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231782.094694:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761231787.214003:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231787.214007:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231787.214508:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231787.214510:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231787.214781:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231787.214784:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761231792.333925:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231792.333931:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231792.334395:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231792.334398:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231792.334664:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231792.334666:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231797.453940:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231797.453943:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231797.454380:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231797.454383:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231797.454637:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231797.454640:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231802.574136:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231802.574163:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231802.574872:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231802.574874:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231802.575124:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231802.575126:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761231812.814777:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761231817.934189:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231817.934196:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761231817.934941:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231817.934943:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761231817.935222:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231817.935224:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761231823.053734:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231823.054041:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231823.054212:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231828.174065:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231828.174070:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231828.174643:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231828.174647:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231828.175155:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231828.175158:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761231833.293964:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231833.293975:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761231833.294354:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231833.294356:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761231833.294559:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231833.294561:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761231838.413766:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231838.413773:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761231838.414268:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231838.414271:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761231838.414609:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231838.414611:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761231843.533760:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231843.533764:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761231843.534187:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231843.534189:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761231843.534554:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231843.534557:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761231848.654042:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231848.654046:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761231848.654421:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231848.654425:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761231848.654894:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231848.654897:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761231853.773775:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231853.773779:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761231853.774194:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231853.774196:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761231853.774477:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231853.774479:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761231858.893997:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231858.894001:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761231858.894439:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231858.894442:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761231858.894760:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231858.894763:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761231864.014207:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231864.014217:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761231864.014746:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231864.014749:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761231864.015065:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231864.015068:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761231869.133761:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231869.133764:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761231869.134049:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231869.134050:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761231869.134242:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231869.134244:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761231874.253815:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231874.253819:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761231874.254306:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231874.254309:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761231874.254565:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231874.254567:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231879.373965:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231879.373969:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231879.374455:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231879.374458:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231879.374699:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231879.374702:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761231884.493960:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231884.493969:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231884.494378:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231884.494381:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231884.494747:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231884.494750:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761231889.614090:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231889.614310:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231889.614501:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231894.734296:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231894.734309:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761231894.735098:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231894.735102:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761231894.735543:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231894.735546:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231899.854134:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231899.854146:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231899.854585:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231899.854590:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231899.854939:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231899.854941:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761231904.973951:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231904.973960:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761231904.974831:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231904.974833:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761231904.975140:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231904.975142:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761231910.093837:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231910.093841:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1761231910.094331:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231910.094334:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1761231910.094631:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231910.094634:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761231915.213824:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231915.213827:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761231915.214237:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231915.214240:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761231915.214513:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231915.214516:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761231920.333966:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231920.333970:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761231920.334365:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231920.334367:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761231920.334651:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231920.334652:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761231925.454167:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231925.454173:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761231925.454651:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231925.454654:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761231925.454963:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231925.454965:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761231930.573992:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231930.573997:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231930.574455:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231930.574458:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231930.574722:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231930.574725:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761231935.693579:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231935.693583:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231935.693998:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231935.694000:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231935.694244:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231935.694246:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761231940.813472:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231940.813475:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231940.813647:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231940.813648:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231940.813893:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231940.813895:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761231945.933548:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231945.933552:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1761231945.933816:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231945.933817:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1761231945.933993:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231945.933994:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1761231951.053587:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231951.053590:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761231951.053760:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231951.053761:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761231951.053993:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231951.053995:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761231956.173784:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231956.174125:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231956.174294:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231961.294402:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231961.294407:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231961.295017:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231961.295020:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761231961.295672:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761231961.295676:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761231966.414159:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231966.414165:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761231966.414608:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231966.414611:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761231966.414934:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231966.414937:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761231971.533980:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231971.533983:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761231971.534293:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231971.534295:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761231971.534653:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231971.534656:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761231976.653834:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231976.653839:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761231976.654325:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231976.654327:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761231976.654585:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231976.654588:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761231981.774035:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231981.774040:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761231981.774404:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231981.774406:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761231981.774672:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231981.774674:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761231986.893835:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231986.893840:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761231986.894306:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231986.894308:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761231986.894569:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231986.894572:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761231992.013763:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231992.013767:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761231992.014202:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231992.014205:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761231992.014587:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231992.014590:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761231997.134147:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231997.134152:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761231997.134752:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231997.134755:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761231997.135222:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761231997.135225:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232002.253953:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232002.253958:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232002.254550:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232002.254554:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232002.255000:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232002.255004:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232007.373633:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232007.373635:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232007.373987:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232007.373990:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232007.374277:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232007.374279:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232012.494240:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232012.494245:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232012.494777:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232012.494780:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232012.495098:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232012.495102:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232017.613819:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232017.613827:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232017.614208:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232017.614209:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232017.614438:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232017.614440:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232022.733985:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232022.734362:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232022.734454:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232027.854278:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232027.854290:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232027.855044:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232027.855049:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232027.855517:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232027.855522:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232032.973949:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232032.973953:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1761232032.974364:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232032.974367:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1761232032.974630:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232032.974633:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1761232038.094036:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232038.094040:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761232038.094455:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232038.094458:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761232038.094708:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232038.094710:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761232043.213992:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232043.213996:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1761232043.214348:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232043.214350:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1761232043.214717:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232043.214719:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1761232048.333780:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232048.333784:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761232048.334179:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232048.334181:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761232048.334530:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232048.334532:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761232053.454006:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232053.454010:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761232053.454325:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232053.454327:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761232053.454708:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232053.454711:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761232058.573985:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232058.573988:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761232058.574320:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232058.574322:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761232058.574669:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232058.574671:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761232063.693764:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232063.693767:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761232063.694277:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232063.694279:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761232063.694483:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232063.694485:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761232068.813895:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232068.813900:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761232068.814312:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232068.814315:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761232068.814578:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232068.814581:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761232073.933783:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232073.933792:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1761232073.934440:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232073.934443:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1761232073.934731:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232073.934734:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1761232079.053710:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232079.053713:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1761232079.054136:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232079.054138:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1761232079.054389:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232079.054391:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1761232084.173707:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232084.173711:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761232084.174134:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232084.174136:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761232084.174394:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232084.174396:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761232089.293507:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232089.293677:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232089.293795:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232094.413948:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232094.413952:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232094.414346:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232094.414348:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232094.414633:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232094.414636:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232099.533526:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232099.533529:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232099.533706:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232099.533708:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232099.533942:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232099.533943:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232104.653979:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232104.653988:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232104.654579:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232104.654581:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232104.654882:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232104.654884:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232109.774037:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232109.774041:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761232109.774493:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232109.774496:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761232109.774756:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232109.774759:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761232114.893700:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232114.893704:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761232114.894201:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232114.894205:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761232114.894549:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232114.894552:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1761232120.013647:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232120.013651:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761232120.013939:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232120.013941:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761232120.014089:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232120.014091:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1761232125.133763:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232125.133766:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761232125.134075:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232125.134077:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761232125.134276:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232125.134277:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761232130.254196:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232130.254199:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761232130.254582:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232130.254584:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761232130.254843:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232130.254876:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761232135.373620:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232135.373624:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761232135.374023:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232135.374026:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761232135.374257:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232135.374259:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761232140.494207:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232140.494211:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761232140.494603:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232140.494606:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761232140.494887:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232140.494889:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761232145.614180:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232145.614185:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761232145.614581:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232145.614584:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761232145.615055:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232145.615057:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761232150.734164:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232150.734169:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761232150.734550:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232150.734552:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761232150.734842:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232150.734875:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761232155.853727:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232155.854098:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232155.854258:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232160.973587:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232160.973590:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232160.973957:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232160.973960:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232160.974306:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232160.974310:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232166.093746:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232166.093750:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232166.094196:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232166.094198:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232166.094449:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232166.094452:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232171.213927:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232171.213936:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232171.214462:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232171.214465:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232171.214745:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232171.214747:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232176.333602:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232176.333607:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761232176.334121:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232176.334124:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761232176.334380:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232176.334382:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1761232181.454218:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232181.454223:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761232181.454679:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232181.454682:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761232181.454984:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232181.454987:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761232186.574011:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232186.574015:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1761232186.574356:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232186.574359:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1761232186.574786:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232186.574789:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761232191.693981:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232191.693989:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761232191.694886:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232191.694888:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761232191.695113:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232191.695115:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761232196.814210:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232196.814216:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1761232196.814648:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232196.814650:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1761232196.814943:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232196.814946:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1761232201.933961:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232201.933968:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232201.934675:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232201.934679:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232201.935186:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232201.935190:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232207.053966:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232207.053970:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232207.054384:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232207.054386:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232207.054645:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232207.054647:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232212.174038:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232212.174042:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232212.174483:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232212.174486:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232212.174749:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232212.174751:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761232217.293967:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232217.293978:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761232217.294460:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232217.294463:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761232217.294750:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232217.294752:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761232222.413893:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232222.414182:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232222.414371:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232227.533576:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232227.533582:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232227.534344:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232227.534348:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232227.534834:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232227.534838:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232232.653943:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232232.653949:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232232.654519:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232232.654523:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232232.654988:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232232.654994:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232237.773739:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232237.773745:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232237.774402:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232237.774405:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232237.774676:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232237.774679:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232242.894022:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232242.894027:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232242.894437:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232242.894440:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232242.894699:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232242.894701:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232248.014087:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232248.014093:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232248.014541:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232248.014543:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232248.014791:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232248.014793:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232253.133948:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232253.133958:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232253.134730:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232253.134734:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232253.135212:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232253.135216:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232258.253892:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232258.253897:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232258.254287:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232258.254290:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232258.254545:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232258.254548:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1761232263.373980:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232263.373988:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761232263.374769:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232263.374772:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761232263.375186:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232263.375188:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1761232268.494066:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232268.494072:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761232268.494668:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232268.494670:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761232268.494973:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232268.494976:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1761232273.613570:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232273.613575:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761232273.614060:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232273.614063:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761232273.614323:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232273.614325:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1761232278.734050:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232278.734055:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761232278.734472:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232278.734474:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761232278.734743:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232278.734746:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761232283.854193:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232283.854197:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761232283.854578:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232283.854580:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761232283.855013:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232283.855016:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1761232288.973975:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232288.974184:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232288.974499:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232294.093921:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232294.093926:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232294.094591:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232294.094595:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232294.095175:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232294.095208:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232299.213919:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232299.213923:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232299.214332:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232299.214335:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232299.214596:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232299.214599:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232304.333826:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232304.333830:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232304.334402:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232304.334406:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1761232304.334728:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232304.334730:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232309.453909:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232309.453912:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761232309.454340:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232309.454342:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761232309.454580:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232309.454582:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761232314.573715:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232314.573718:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761232314.574166:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232314.574169:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761232314.574424:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232314.574427:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761232319.693843:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232319.693876:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761232319.694302:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232319.694303:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761232319.694469:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232319.694471:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761232324.813412:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232324.813421:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761232324.813772:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232324.813774:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761232324.814047:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232324.814049:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761232329.933923:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232329.933929:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232329.934384:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232329.934387:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232329.934648:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232329.934650:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232335.054099:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232335.054103:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232335.054486:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232335.054488:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232335.054775:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232335.054777:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232340.173811:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232340.173815:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232340.174141:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232340.174143:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232340.174469:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232340.174471:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232345.293886:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232345.293891:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232345.294235:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232345.294237:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232345.294414:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232345.294416:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232350.413509:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232350.413512:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232350.413789:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232350.413791:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232350.413980:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232350.413982:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232355.534046:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232355.534327:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232355.534555:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232360.653698:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232360.653707:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232360.654599:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232360.654603:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232360.655134:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232360.655138:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232365.773544:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232365.773551:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232365.773953:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232365.773955:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232365.774195:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232365.774197:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1761232370.893808:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232370.893812:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761232370.894265:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232370.894267:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1761232370.894558:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232370.894560:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1761232376.013429:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232376.013431:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232376.013669:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232376.013671:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232376.013998:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232376.014000:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232381.133571:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232381.133575:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232381.133944:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232381.133946:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232381.134273:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232381.134274:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232386.253793:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232386.253796:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232386.254221:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232386.254223:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232386.254467:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232386.254469:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232391.373831:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232391.373835:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232391.374290:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232391.374293:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232391.374547:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232391.374550:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232396.493911:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232396.493914:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232396.494318:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232396.494321:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232396.494577:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232396.494579:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232401.613901:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232401.613904:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232401.614335:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232401.614337:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232401.614553:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232401.614556:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232406.733501:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232406.733507:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232406.733932:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232406.733933:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232406.734072:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232406.734073:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232411.853589:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232411.853591:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232411.853804:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232411.853806:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232411.853999:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232411.854000:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1761232416.973512:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232416.973516:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761232416.973837:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232416.973839:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761232416.974094:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232416.974095:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1761232422.093397:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232422.093500:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232422.093614:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232427.213368:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232427.213371:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232427.213728:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232427.213730:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1761232427.213987:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232427.213988:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232432.333933:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232432.333953:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232432.334651:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232432.334653:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232432.334951:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232432.334953:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232437.453828:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232437.453835:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232437.454336:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232437.454338:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232437.454600:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232437.454603:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232442.573798:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232442.573802:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761232442.574192:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232442.574195:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761232442.574571:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232442.574573:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761232447.694043:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232447.694047:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761232447.694366:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232447.694368:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761232447.694751:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232447.694754:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1761232452.813651:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:0.1:1761232452.813655:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761232452.814138:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232452.814141:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761232452.814657:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232452.814660:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761232457.933651:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232457.933654:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232457.934060:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232457.934062:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232457.934277:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232457.934279:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232463.053656:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232463.053659:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232463.053977:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232463.053978:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232463.054134:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232463.054136:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232468.173736:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232468.173764:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232468.174252:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232468.174255:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232468.174523:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232468.174525:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232473.293442:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232473.293446:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761232473.293733:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232473.293734:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761232473.293913:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232473.293914:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761232478.413699:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232478.413706:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232478.414271:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232478.414273:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232478.414553:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232478.414556:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232483.533785:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232483.533788:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232483.534200:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232483.534202:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232483.534421:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232483.534423:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232488.654150:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232488.654424:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232488.654642:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232493.774222:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232493.774227:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761232493.774950:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232493.774955:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761232493.775403:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232493.775407:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761232498.893806:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232498.893810:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1761232498.894259:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232498.894262:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1761232498.894522:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232498.894525:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1761232504.013596:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232504.013598:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1761232504.013831:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232504.013833:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1761232504.014131:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232504.014133:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1761232509.133415:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232509.133418:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761232509.133648:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232509.133649:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761232509.133818:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232509.133831:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761232514.253736:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232514.253739:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761232514.254195:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232514.254199:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761232514.254506:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232514.254508:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761232519.373437:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232519.373439:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232519.373756:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232519.373758:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232519.374113:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232519.374116:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232524.493932:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232524.493936:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232524.494313:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232524.494316:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232524.494592:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232524.494595:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232529.613381:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232529.613383:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232529.613614:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232529.613615:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232529.613747:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232529.613748:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232534.733719:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232534.733723:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232534.734140:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232534.734143:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232534.734401:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232534.734403:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232539.854172:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232539.854180:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232539.854660:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232539.854662:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232539.854956:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232539.854958:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232544.973683:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232544.973690:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1761232544.974195:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232544.974198:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1761232544.974500:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232544.974503:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1761232550.093698:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232550.093702:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761232550.094176:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232550.094178:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761232550.094427:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232550.094429:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761232555.213763:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232555.214078:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232555.214258:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232560.333991:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232560.333996:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761232560.334611:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232560.334616:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761232560.335144:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232560.335147:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761232565.453899:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232565.453902:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761232565.454313:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232565.454315:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761232565.454575:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232565.454597:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761232570.573953:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232570.573960:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761232570.574500:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232570.574504:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761232570.574783:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232570.574785:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761232575.693811:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232575.693815:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761232575.694216:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232575.694218:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761232575.694402:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232575.694404:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761232580.813819:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232580.813825:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761232580.814417:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232580.814420:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761232580.814649:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232580.814651:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761232585.933774:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232585.933778:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232585.934270:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232585.934273:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232585.934526:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232585.934549:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232591.054012:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232591.054033:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232591.054444:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232591.054446:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232591.054712:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232591.054714:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232596.173553:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232596.173556:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232596.173964:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232596.173966:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232596.174202:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232596.174204:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232601.293532:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232601.293536:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232601.293798:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232601.293800:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232601.293994:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232601.293996:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232606.414001:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232606.414006:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761232606.414365:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232606.414367:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761232606.414789:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232606.414813:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761232611.533893:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232611.533897:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232611.534399:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232611.534401:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232611.534700:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232611.534702:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232616.653965:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232616.653968:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232616.654289:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232616.654292:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232616.654688:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232616.654691:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232621.773836:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232621.774270:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232621.774422:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232626.893666:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232626.893670:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1761232626.894056:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232626.894059:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1761232626.894351:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232626.894354:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1761232632.013833:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232632.013837:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1761232632.014266:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232632.014289:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1761232632.014587:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232632.014590:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1761232637.133942:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232637.133946:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1761232637.134355:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232637.134357:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1761232637.134620:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232637.134623:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1761232642.253810:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232642.253814:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761232642.254172:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232642.254175:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761232642.254559:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232642.254562:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761232647.373390:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232647.373392:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761232647.373592:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232647.373594:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761232647.373724:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232647.373725:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761232652.493812:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232652.493817:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232652.494485:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232652.494509:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232652.494998:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232652.495002:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232657.613997:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232657.614002:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232657.614495:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232657.614498:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232657.614721:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232657.614723:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232662.733826:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232662.733835:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232662.734317:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232662.734320:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232662.734606:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232662.734608:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232667.853805:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232667.853811:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232667.854439:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232667.854442:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232667.854912:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232667.854914:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232672.973326:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232672.973329:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1761232672.973493:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232672.973495:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1761232672.973672:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232672.973673:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1761232678.093509:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232678.093513:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1761232678.093779:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232678.093781:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1761232678.094091:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232678.094094:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1761232683.213502:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232683.213505:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761232683.213741:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232683.213742:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761232683.213958:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232683.213960:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761232688.333751:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232688.334102:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232688.334242:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232693.453967:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232693.453973:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232693.454614:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232693.454617:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232693.455151:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232693.455154:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232698.573648:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232698.573651:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232698.574042:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232698.574044:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232698.574621:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232698.574624:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232703.693359:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232703.693362:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232703.693577:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232703.693578:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232703.693763:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232703.693764:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232708.813692:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232708.813698:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761232708.814317:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232708.814320:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761232708.814660:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232708.814661:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761232713.933559:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232713.933561:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232713.933778:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232713.933779:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232713.933954:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232713.933955:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232719.053507:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232719.053510:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232719.053759:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232719.053761:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232719.054019:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232719.054021:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232724.173433:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232724.173436:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232724.173677:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232724.173678:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232724.173826:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232724.173827:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761232729.293643:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232729.293648:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232729.293994:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232729.293996:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232729.294189:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232729.294191:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761232734.413575:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232734.413580:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232734.414002:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232734.414004:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232734.414242:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232734.414244:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232739.533609:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232739.533611:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761232739.533830:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232739.533831:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761232739.533990:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232739.533992:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232744.653453:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232744.653456:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232744.653717:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232744.653718:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232744.653872:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232744.653873:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232749.773895:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232749.773903:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232749.774358:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232749.774361:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232749.774617:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232749.774619:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232754.893830:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232754.894156:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232754.894296:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232760.014133:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232760.014141:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232760.014677:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232760.014679:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232760.014993:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232760.014995:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232765.133553:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232765.133559:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232765.133951:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232765.133953:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232765.134086:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232765.134087:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761232770.253982:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232770.254013:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761232770.254446:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232770.254448:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761232770.254713:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232770.254715:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761232775.373916:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232775.373919:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761232775.374192:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232775.374193:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761232775.374337:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232775.374338:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761232780.494217:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232780.494224:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761232780.494736:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232780.494738:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761232780.494977:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232780.494979:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761232785.613978:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232785.613981:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761232785.614359:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232785.614361:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761232785.614648:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232785.614650:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761232790.733952:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232790.733957:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761232790.734362:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232790.734364:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761232790.734669:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232790.734671:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761232795.853795:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232795.853798:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1761232795.854195:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232795.854215:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1761232795.854467:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232795.854490:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1761232800.973576:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232800.973578:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232800.973890:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232800.973892:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232800.974087:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232800.974089:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232806.094128:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232806.094137:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232806.094550:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232806.094552:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232806.094754:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232806.094755:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232811.213572:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232811.213575:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232811.213931:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232811.213933:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232811.214237:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232811.214240:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232816.333891:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232816.333895:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232816.334271:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232816.334274:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232816.334503:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232816.334505:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232821.453567:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232821.453741:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232821.453885:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000010:0.0:1761232824.525485:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009b62b3c1 (tot 1676530). 00000400:00000200:0.0:1761232824.525489:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761232824.525492:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761232824.525495:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000ca0eefa8 00000400:00000010:0.0:1761232824.525497:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000ca0eefa8. 00000100:00000001:0.0:1761232824.525499:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761232824.525503:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000db341c88 x1846784245180928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761232846 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232824.525510:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761232824.525511:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000db341c88 x1846784245180928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761232846 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232824.525516:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232824.525517:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761232824.525520:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761232824.525521:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:0.0:1761232824.525524:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761232824.525526:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761232824.525529:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:0.0:1761232824.525531:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:0.0:1761232824.525533:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761232824.525537:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000266eaa93 x1846784245180992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761232846 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232824.525544:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761232824.525546:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000266eaa93 x1846784245180992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761232846 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232824.525551:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232824.525553:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000200:2.1:1761232826.574304:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232826.574310:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232826.575092:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232826.575096:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232826.575574:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232826.575578:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232831.694215:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232831.694221:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232831.694606:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232831.694609:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232831.694891:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232831.694893:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000020:00000001:0.0:1761232832.589281:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761232832.589287:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761232832.589294:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761232836.813802:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232836.813807:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232836.814353:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232836.814355:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761232836.814656:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232836.814658:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000100:00080000:0.0:1761232841.933071:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232841.933084:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232841.933086:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761232841.933091:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232841.933095:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232841.933101:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232841.933109:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5310 00000100:00080000:0.0:1761232841.933113:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232841.933117:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933120:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232841.933123:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933125:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232841.933130:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232841.933133:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761232841.933136:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232841.933139:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761232841.933142:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761232841.933145:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761232841.933148:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933150:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232841.933153:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933155:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232841.933158:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761232841.933161:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933164:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232841.933167:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232841.933170:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232841.933174:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933176:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761232841.933179:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232841.933187:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000006edc786f. 00000020:00000040:0.0:1761232841.933191:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761232841.933196:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232841.933197:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933201:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232841.933205:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232841.933210:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232841.933219:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761232841.933227:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232841.933236:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006edc786f x1846784245181632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232841.933248:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232841.933251:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232841.933255:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000006edc786f] to pc [ptlrpcd_rcv+-1] req@000000006edc786f x1846784245181632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232841.933283:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933286:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761232841.933288:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232841.933292:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232841.933293:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761232841.933295:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232841.933296:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232841.933300:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232841.933303:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5310 00000100:00080000:0.0:1761232841.933305:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232841.933308:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933310:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232841.933312:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933315:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232841.933317:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232841.933318:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761232841.933320:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232841.933322:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761232841.933324:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761232841.933327:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761232841.933330:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933333:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232841.933335:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933338:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232841.933340:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761232841.933343:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933346:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232841.933348:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232841.933354:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232841.933356:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933358:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761232841.933362:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761232841.933366:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933369:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232841.933374:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ad5ae63. 00000020:00000040:0.0:1761232841.933377:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761232841.933380:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232841.933381:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933384:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232841.933386:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232841.933388:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232841.933392:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761232841.933396:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232841.933402:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ad5ae63 x1846784245181696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232841.933415:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232841.933417:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232841.933427:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ad5ae63] to pc [ptlrpcd_rcv+-1] req@000000009ad5ae63 x1846784245181696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232841.933469:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933474:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761232841.933477:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232841.933484:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232841.933488:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761232841.933493:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232841.933495:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232841.933500:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232841.933508:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5310 00000100:00080000:0.0:1761232841.933512:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232841.933517:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933521:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232841.933527:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933531:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232841.933534:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232841.933536:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761232841.933540:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232841.933544:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761232841.933551:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761232841.933553:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761232841.933558:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933564:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232841.933566:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933569:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232841.933575:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761232841.933579:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761232841.933582:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232841.933588:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232841.933594:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232841.933600:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933603:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761232841.933607:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761232841.933614:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933619:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232841.933628:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761232841.933636:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761232841.933642:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232841.933645:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933648:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232841.933653:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232841.933657:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232841.933665:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761232841.933671:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232841.933681:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245181760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232841.933690:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232841.933692:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232841.933700:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245181760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232841.933716:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232841.933720:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761232841.933725:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5321) 00000800:00000200:2.1:1761232841.933792:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232841.933796:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232841.934312:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232841.934315:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232841.934572:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232841.934574:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761232847.053817:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232847.053820:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761232847.054221:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232847.054223:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761232847.054462:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232847.054463:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761232852.173545:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232852.173561:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232852.173797:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232852.173799:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232852.174018:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232852.174020:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232857.293938:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232857.293943:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232857.294349:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232857.294352:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232857.294606:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232857.294608:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232862.414020:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232862.414023:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232862.414313:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232862.414315:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232862.414637:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232862.414639:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232867.533911:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232867.533915:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232867.534491:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232867.534495:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232867.534968:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232867.534972:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1761232868.045105:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232868.045113:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232868.045116:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232868.045123:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232868.045133:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232868.045140:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232868.045144:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232868.045147:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232868.045151:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232869.069004:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232869.069011:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232869.069013:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232869.069019:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232869.069025:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232869.069029:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232869.069032:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232869.069035:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232869.069038:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232870.092933:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232870.092939:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232870.092940:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232870.092945:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232870.092950:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232870.092954:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232870.092956:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232870.092959:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232870.092961:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232871.116969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232871.116975:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232871.116976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232871.116982:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232871.116987:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232871.116992:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232871.116995:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232871.116997:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232871.117000:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232872.140967:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232872.140984:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232872.140986:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232872.140992:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232872.140999:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232872.141003:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232872.141006:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232872.141009:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232872.141012:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761232872.654098:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232872.654102:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232872.654420:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232872.654422:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232872.654793:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232872.654795:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000100:00000001:0.0:1761232873.165007:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232873.165013:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232873.165015:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232873.165020:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232873.165026:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232873.165030:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232873.165033:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232873.165036:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232873.165040:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232874.188969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232874.188976:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232874.188978:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232874.188984:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232874.188990:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232874.188994:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232874.188998:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232874.189001:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232874.189004:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232875.212954:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232875.212961:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232875.212963:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232875.212969:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232875.212976:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232875.212980:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232875.212983:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232875.212986:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232875.212990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232876.236995:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232876.237003:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232876.237004:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232876.237009:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232876.237016:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232876.237020:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232876.237023:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232876.237025:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232876.237028:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.260998:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232877.261004:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232877.261006:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232877.261011:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232877.261017:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232877.261023:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232877.261025:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232877.261028:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232877.261030:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1761232877.773091:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232877.773104:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232877.773107:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761232877.773110:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232877.773114:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232877.773119:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232877.773127:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5346 00000100:00080000:0.0:1761232877.773130:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232877.773134:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773137:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232877.773139:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773141:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232877.773145:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232877.773147:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761232877.773150:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232877.773153:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761232877.773156:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761232877.773159:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761232877.773162:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773164:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232877.773166:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773167:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232877.773170:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761232877.773174:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773176:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232877.773179:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232877.773182:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232877.773185:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773188:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761232877.773190:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232877.773201:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761232877.773204:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761232877.773209:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232877.773210:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773214:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232877.773217:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232877.773221:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232877.773230:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761232877.773238:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232877.773245:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245182976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232877.773256:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232877.773259:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232877.773263:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245182976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232877.773296:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773299:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761232877.773301:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232877.773304:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232877.773305:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761232877.773307:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232877.773309:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232877.773312:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232877.773314:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5346 00000100:00080000:0.0:1761232877.773316:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232877.773318:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773320:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232877.773322:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773324:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232877.773326:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232877.773327:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761232877.773328:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232877.773330:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761232877.773332:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761232877.773334:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761232877.773336:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773339:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232877.773341:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773343:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232877.773345:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761232877.773347:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773349:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232877.773351:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232877.773356:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232877.773358:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773360:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761232877.773363:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761232877.773366:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773368:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232877.773371:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761232877.773373:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761232877.773376:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232877.773377:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773379:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232877.773380:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232877.773382:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232877.773388:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761232877.773392:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232877.773397:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245183040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232877.773406:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232877.773408:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232877.773413:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245183040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232877.773421:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773426:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761232877.773428:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232877.773435:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232877.773439:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761232877.773442:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232877.773444:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232877.773452:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232877.773460:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5346 00000100:00080000:0.0:1761232877.773466:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232877.773473:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773479:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232877.773484:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773487:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232877.773492:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232877.773494:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761232877.773498:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232877.773501:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761232877.773507:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761232877.773513:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761232877.773522:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773528:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232877.773534:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773560:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232877.773563:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761232877.773568:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761232877.773578:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232877.773582:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232877.773586:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232877.773591:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773593:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761232877.773596:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761232877.773601:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773604:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232877.773610:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761232877.773617:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761232877.773623:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232877.773625:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773628:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232877.773631:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232877.773633:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232877.773640:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761232877.773647:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232877.773660:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245183104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232877.773678:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232877.773681:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232877.773689:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245183104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232877.773701:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232877.773703:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761232877.773708:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5356) 00000800:00000200:2.1:1761232877.773783:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232877.773786:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232877.774123:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232877.774126:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761232877.774451:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232877.774453:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000100:00000001:0.0:1761232878.285024:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232878.285030:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232878.285032:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232878.285038:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232878.285044:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232878.285048:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232878.285051:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232878.285054:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232878.285057:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232879.308972:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232879.308977:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232879.308979:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232879.308983:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232879.308989:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232879.308993:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232879.308995:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232879.308998:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232879.309000:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232880.332909:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232880.332915:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232880.332918:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232880.332923:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232880.332929:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232880.332933:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232880.332936:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232880.332939:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232880.332942:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232881.356953:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232881.356960:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232881.356961:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232881.356967:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232881.356972:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232881.356976:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232881.356978:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232881.356981:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232881.356984:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232882.380993:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232882.380999:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232882.381001:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232882.381007:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232882.381013:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232882.381019:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232882.381021:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232882.381024:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232882.381028:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761232882.893646:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232882.893649:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761232882.894016:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232882.894018:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761232882.894463:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232882.894466:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000100:00000001:0.0:1761232883.404940:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232883.404946:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232883.404948:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232883.404953:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232883.404958:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232883.404961:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232883.404963:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232883.404966:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232883.404969:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232884.428960:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232884.428964:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232884.428965:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232884.428970:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232884.428974:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232884.428978:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232884.428980:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232884.428982:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232884.428984:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232885.453003:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232885.453008:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232885.453009:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232885.453013:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232885.453018:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232885.453021:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232885.453024:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232885.453026:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232885.453028:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232886.476979:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232886.476985:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232886.476987:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232886.476993:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232886.476999:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232886.477004:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232886.477007:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232886.477010:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232886.477013:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232887.500982:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232887.500990:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232887.500992:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232887.500998:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232887.501006:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232887.501010:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232887.501013:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232887.501016:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232887.501019:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761232888.013663:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232888.013917:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761232888.013939:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000400:00000200:0.0:1761232888.014076:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000800:00000200:2.1:1761232888.014125:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761232888.014128:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000100:00000001:0.0:1761232888.524972:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232888.524979:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232888.524981:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232888.524986:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232888.524992:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232888.524996:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232888.524999:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232888.525002:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232888.525005:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232889.548974:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232889.548981:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232889.548983:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232889.548988:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232889.548994:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232889.548997:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232889.549000:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232889.549003:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232889.549006:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232890.572967:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232890.572971:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232890.572973:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232890.572977:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232890.572980:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232890.572983:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232890.572985:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232890.572987:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232890.572989:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1761232891.085055:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:0.0:1761232891.085094:0:88982:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1761232891.085100:0:88982:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:0.0:1761232891.085109:0:88982:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761232891.085112:0:88982:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007ceb0618 (tot 1676778). 00000400:00000200:0.0:1761232891.085117:0:88982:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761232891.085121:0:88982:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761232891.085125:0:88982:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000002b6caf4f 00000400:00000010:0.0:1761232891.085127:0:88982:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000002b6caf4f. 00000100:00000001:0.0:1761232891.085131:0:88982:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761232891.085136:0:88982:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000108ab0a5 x1846784245183360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761232913 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232891.085156:0:88982:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761232891.085158:0:88982:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000108ab0a5 x1846784245183360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761232913 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232891.085166:0:88982:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232891.085168:0:88982:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761232891.085175:0:88982:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761232891.085176:0:88982:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676530). 00000400:00000200:0.0:1761232891.085178:0:88982:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761232891.085180:0:88982:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761232891.085182:0:88982:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000001d64cf84 00000400:00000010:0.0:1761232891.085183:0:88982:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000001d64cf84. 00000100:00000001:0.0:1761232891.085185:0:88982:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761232891.085187:0:88982:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000f61de4c8 x1846784245183424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761232913 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232891.085191:0:88982:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761232891.085193:0:88982:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f61de4c8 x1846784245183424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761232913 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232891.085197:0:88982:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232891.085198:0:88982:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761232891.085203:0:88982:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761232891.085205:0:88982:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:0.0:1761232891.085208:0:88982:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761232891.085211:0:88982:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761232891.085214:0:88982:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000f98e6c57 00000400:00000010:0.0:1761232891.085216:0:88982:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000f98e6c57. 00000100:00000001:0.0:1761232891.085218:0:88982:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761232891.085223:0:88982:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000002431276f x1846784245183488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761232913 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232891.085245:0:88982:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761232891.085248:0:88982:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002431276f x1846784245183488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761232913 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232891.085253:0:88982:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232891.085256:0:88982:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:0.0:1761232891.596923:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232891.596928:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232891.596929:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232891.596932:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232891.596936:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232891.596939:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232891.596941:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232891.596943:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232891.596945:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232892.620975:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232892.620980:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232892.620982:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232892.620986:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232892.620989:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232892.620992:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232892.620994:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232892.620996:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232892.620998:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1761232893.133717:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232893.133741:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761232893.134148:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232893.134150:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761232893.134413:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232893.134415:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1761232893.644986:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232893.644991:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232893.644993:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232893.644998:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232893.645004:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232893.645007:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232893.645010:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232893.645012:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232893.645015:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232895.692984:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232895.692989:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232895.692991:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232895.692995:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232895.693000:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232895.693003:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232895.693005:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232895.693009:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232895.693011:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232896.716936:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232896.716941:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232896.716942:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232896.716946:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232896.716951:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232896.716954:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232896.716955:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232896.716958:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232896.716960:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232897.740967:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761232897.740976:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761232897.740978:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232897.740985:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761232897.740991:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761232897.740994:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232897.740997:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761232897.741000:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761232897.741003:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1761232898.253607:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232898.253614:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761232898.253929:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232898.253930:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761232898.254072:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232898.254073:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761232903.373641:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232903.373647:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761232903.374067:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232903.374070:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761232903.374307:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232903.374309:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761232908.493420:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232908.493424:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761232908.493702:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232908.493703:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761232908.493833:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232908.493835:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761232913.613551:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232913.613573:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761232913.613936:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232913.613938:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761232913.614103:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232913.614105:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000100:00080000:0.0:1761232918.732982:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232918.732994:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232918.732996:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761232918.732999:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232918.733002:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232918.733008:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232918.733015:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5387 00000100:00080000:0.0:1761232918.733018:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232918.733020:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733023:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232918.733024:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733026:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232918.733029:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232918.733031:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761232918.733034:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232918.733038:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761232918.733039:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761232918.733042:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761232918.733043:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733045:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232918.733046:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733047:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232918.733048:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761232918.733051:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733052:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232918.733055:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232918.733058:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232918.733060:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733062:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761232918.733063:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232918.733098:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761232918.733101:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761232918.733105:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232918.733106:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733109:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232918.733112:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232918.733116:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232918.733125:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761232918.733131:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232918.733138:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245184512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232918.733146:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232918.733148:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232918.733151:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245184512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232918.733169:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733170:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761232918.733172:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232918.733175:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232918.733176:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761232918.733177:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232918.733178:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232918.733180:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232918.733181:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5387 00000100:00080000:0.0:1761232918.733182:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232918.733184:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733185:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232918.733186:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733187:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232918.733189:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232918.733191:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761232918.733193:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232918.733195:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761232918.733196:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761232918.733198:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761232918.733201:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733203:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232918.733205:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733205:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232918.733207:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761232918.733209:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733210:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232918.733211:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232918.733217:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232918.733220:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733222:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761232918.733229:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761232918.733233:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733235:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232918.733238:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761232918.733241:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761232918.733246:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232918.733247:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733254:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232918.733255:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232918.733257:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232918.733265:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761232918.733271:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232918.733277:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245184576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232918.733303:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232918.733305:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232918.733313:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245184576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232918.733322:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733324:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761232918.733326:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232918.733331:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232918.733333:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761232918.733336:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232918.733337:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232918.733340:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232918.733344:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5387 00000100:00080000:0.0:1761232918.733351:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232918.733359:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733364:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232918.733372:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733379:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232918.733384:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232918.733385:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761232918.733387:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232918.733389:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761232918.733392:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761232918.733394:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761232918.733399:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733402:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232918.733406:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733411:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232918.733416:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761232918.733420:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761232918.733424:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232918.733427:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232918.733429:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232918.733433:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733436:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761232918.733438:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761232918.733440:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733446:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232918.733451:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761232918.733452:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761232918.733461:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232918.733463:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733465:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232918.733466:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232918.733468:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232918.733472:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761232918.733475:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232918.733480:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245184640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232918.733493:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232918.733496:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232918.733504:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245184640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232918.733532:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232918.733538:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761232918.733546:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5397) 00000800:00000200:1.1:1761232918.733582:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232918.733584:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761232918.733796:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232918.733798:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761232918.733957:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232918.733958:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761232923.853667:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232923.853671:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1761232923.854072:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232923.854074:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1761232923.854447:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232923.854449:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000100:00080000:0.0:1761232928.973149:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232928.973164:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232928.973167:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761232928.973170:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232928.973174:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232928.973182:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232928.973192:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5397 00000100:00080000:0.0:1761232928.973197:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232928.973201:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973205:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232928.973208:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973210:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232928.973215:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232928.973218:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761232928.973222:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232928.973226:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761232928.973229:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761232928.973232:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761232928.973235:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973237:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232928.973239:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973241:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232928.973244:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761232928.973249:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973251:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232928.973257:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232928.973261:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232928.973265:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973267:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761232928.973270:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232928.973284:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761232928.973289:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761232928.973295:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232928.973297:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973302:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232928.973306:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232928.973312:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232928.973323:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761232928.973332:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232928.973341:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245184896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232928.973355:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232928.973357:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232928.973362:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245184896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232928.973392:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973394:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761232928.973397:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232928.973401:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232928.973403:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761232928.973404:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232928.973406:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232928.973409:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232928.973412:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5397 00000100:00080000:0.0:1761232928.973414:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232928.973417:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973419:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232928.973420:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973423:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232928.973426:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232928.973427:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761232928.973429:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232928.973431:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761232928.973433:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761232928.973435:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761232928.973438:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973440:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232928.973442:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973444:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232928.973446:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761232928.973448:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973450:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232928.973452:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232928.973460:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232928.973463:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973465:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761232928.973470:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761232928.973474:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973476:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232928.973481:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761232928.973484:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761232928.973487:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232928.973488:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973490:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232928.973491:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232928.973494:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232928.973497:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761232928.973504:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232928.973516:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245184960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232928.973550:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761232928.973553:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232928.973570:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245184960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232928.973585:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973591:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761232928.973594:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761232928.973600:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761232928.973606:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761232928.973609:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761232928.973613:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761232928.973619:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761232928.973624:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5397 00000100:00080000:0.0:1761232928.973627:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761232928.973635:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973641:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232928.973647:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973652:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761232928.973656:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761232928.973659:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761232928.973663:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761232928.973674:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761232928.973686:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761232928.973694:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761232928.973711:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973717:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761232928.973722:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973731:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761232928.973738:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761232928.973751:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761232928.973757:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761232928.973762:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761232928.973768:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761232928.973772:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973775:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761232928.973777:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761232928.973780:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973784:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761232928.973790:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761232928.973796:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761232928.973802:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761232928.973805:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.973812:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761232928.973816:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761232928.973819:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761232928.973828:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761232928.973835:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761232928.973913:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245185024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:1.1:1761232928.973940:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:0.0:1761232928.973951:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1761232928.973952:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1761232928.973986:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761232928.973995:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245185024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761232928.974009:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761232928.974015:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761232928.974021:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5408) 00000800:00000200:1.1:1761232928.974226:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232928.974227:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232928.974363:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232928.974365:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232934.093720:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232934.093742:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232934.094176:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232934.094179:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761232934.094736:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232934.094738:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761232939.213482:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232939.213486:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232939.213801:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232939.213836:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761232939.214088:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232939.214090:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761232944.333585:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232944.333588:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232944.333961:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232944.333963:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232944.334180:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232944.334182:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761232949.453452:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232949.453455:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232949.453634:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232949.453635:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761232949.453892:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232949.453893:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761232954.573689:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232954.573901:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232954.574073:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232959.693451:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232959.693454:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232959.693821:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232959.693823:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232959.694095:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232959.694097:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761232964.813825:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232964.813828:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232964.814185:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232964.814187:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232964.814419:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232964.814421:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761232969.933465:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232969.933468:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1761232969.933677:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232969.933679:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1761232969.933812:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232969.933813:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1761232975.053690:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232975.053692:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232975.053944:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232975.053946:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232975.054209:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232975.054210:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761232980.173774:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232980.173776:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232980.174024:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232980.174025:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232980.174255:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232980.174256:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761232985.293659:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232985.293664:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232985.294164:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232985.294166:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761232985.294478:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761232985.294482:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761232990.413798:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232990.413803:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232990.414305:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232990.414308:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232990.414568:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232990.414570:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761232995.534096:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232995.534119:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232995.534517:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232995.534519:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761232995.534776:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761232995.534779:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761233000.653764:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233000.653768:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761233000.654065:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233000.654067:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761233000.654386:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233000.654387:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761233005.773411:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233005.773415:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233005.773726:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233005.773727:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233005.773928:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233005.773929:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233010.893460:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233010.893463:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1761233010.893716:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233010.893718:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1761233010.893884:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233010.893885:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1761233016.013557:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233016.013574:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1761233016.013819:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233016.013820:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1761233016.014106:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233016.014108:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000100:00080000:0.0:1761233021.133036:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233021.133045:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233021.133047:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233021.133049:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233021.133051:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233021.133056:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233021.133062:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5490 00000100:00080000:0.0:1761233021.133064:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233021.133067:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133068:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233021.133070:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133071:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233021.133074:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233021.133075:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233021.133077:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233021.133080:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233021.133081:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233021.133083:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233021.133093:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133094:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233021.133095:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133096:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233021.133098:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233021.133101:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133102:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233021.133104:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233021.133107:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233021.133108:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133110:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233021.133111:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233021.133119:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233021.133122:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233021.133125:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233021.133125:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133131:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233021.133133:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233021.133136:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233021.133142:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233021.133147:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233021.133152:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245188352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233021.133159:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233021.133161:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233021.133163:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245188352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233021.133182:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133184:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233021.133185:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233021.133188:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233021.133189:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233021.133189:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233021.133190:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233021.133192:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233021.133193:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5490 00000100:00080000:0.0:1761233021.133194:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233021.133195:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133196:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233021.133197:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133198:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233021.133200:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233021.133200:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233021.133201:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233021.133203:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233021.133205:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233021.133206:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233021.133208:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133209:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233021.133211:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133213:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233021.133215:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233021.133219:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133222:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233021.133223:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233021.133228:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233021.133231:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133234:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233021.133237:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233021.133239:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133240:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233021.133242:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233021.133243:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233021.133245:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233021.133245:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133247:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233021.133252:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233021.133255:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233021.133259:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233021.133262:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233021.133268:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245188416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233021.133285:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233021.133286:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233021.133293:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245188416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233021.133299:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133300:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233021.133301:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233021.133303:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233021.133303:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233021.133304:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233021.133305:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233021.133307:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233021.133310:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5490 00000100:00080000:0.0:1761233021.133313:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233021.133319:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133323:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233021.133327:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133330:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233021.133336:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233021.133338:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233021.133342:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233021.133347:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233021.133354:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233021.133357:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233021.133361:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133364:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233021.133369:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133371:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233021.133372:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233021.133374:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233021.133376:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233021.133377:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233021.133382:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233021.133390:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133391:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233021.133392:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233021.133394:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133398:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233021.133401:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233021.133404:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233021.133410:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233021.133412:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133413:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233021.133414:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233021.133415:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233021.133419:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233021.133421:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233021.133424:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245188480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233021.133429:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233021.133433:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233021.133441:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245188480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233021.133460:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233021.133467:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233021.133471:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5500) 00000800:00000200:1.1:1761233021.133606:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233021.133756:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233021.133875:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233026.253618:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233026.253622:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761233026.253966:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233026.253968:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761233026.254299:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233026.254301:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761233031.373331:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233031.373336:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761233031.373618:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233031.373621:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761233031.373789:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233031.373791:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000100:00080000:0.0:1761233036.492979:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233036.492988:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233036.492989:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233036.492991:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233036.492993:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233036.492998:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233036.493003:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5505 00000100:00080000:0.0:1761233036.493005:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233036.493007:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493009:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233036.493010:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493011:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233036.493014:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233036.493016:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233036.493018:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233036.493020:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233036.493022:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233036.493023:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233036.493026:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493027:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233036.493028:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493029:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233036.493031:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233036.493033:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493034:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233036.493036:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233036.493038:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233036.493040:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493041:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233036.493042:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233036.493050:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000095969832. 00000020:00000040:0.0:1761233036.493052:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233036.493056:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233036.493057:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493059:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233036.493061:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233036.493064:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233036.493069:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233036.493074:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233036.493080:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000095969832 x1846784245188928/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233036.493086:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233036.493088:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233036.493090:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000095969832] to pc [ptlrpcd_rcv+-1] req@0000000095969832 x1846784245188928/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233036.493107:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493109:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233036.493110:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233036.493112:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233036.493113:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233036.493114:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233036.493115:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233036.493117:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233036.493118:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5505 00000100:00080000:0.0:1761233036.493120:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233036.493121:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493122:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233036.493123:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493124:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233036.493125:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233036.493125:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233036.493126:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233036.493128:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233036.493129:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233036.493130:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233036.493132:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493133:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233036.493135:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493136:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233036.493137:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233036.493139:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493140:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233036.493142:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233036.493147:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233036.493150:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493152:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233036.493156:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233036.493159:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493161:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233036.493164:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002ca01b78. 00000020:00000040:0.0:1761233036.493167:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233036.493171:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233036.493171:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493174:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233036.493176:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233036.493178:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233036.493185:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233036.493189:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233036.493196:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ca01b78 x1846784245188992/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233036.493203:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233036.493205:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233036.493210:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002ca01b78] to pc [ptlrpcd_rcv+-1] req@000000002ca01b78 x1846784245188992/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233036.493218:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493220:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233036.493221:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233036.493223:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233036.493225:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233036.493227:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233036.493228:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233036.493230:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233036.493233:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5505 00000100:00080000:0.0:1761233036.493235:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233036.493237:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493239:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233036.493241:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493245:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233036.493250:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233036.493252:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233036.493253:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233036.493255:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233036.493258:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233036.493262:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233036.493267:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493270:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233036.493290:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493292:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233036.493293:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233036.493294:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233036.493297:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233036.493299:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233036.493302:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233036.493307:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493309:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233036.493310:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233036.493315:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493318:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233036.493321:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233036.493323:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233036.493325:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233036.493326:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493327:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233036.493329:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233036.493331:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233036.493334:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233036.493337:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233036.493344:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245189056/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233036.493359:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233036.493361:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233036.493366:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245189056/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233036.493380:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233036.493383:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233036.493387:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5515) 00000800:00000200:1.1:1761233036.493402:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233036.493405:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761233036.493681:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233036.493682:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761233036.493930:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233036.493933:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761233041.613583:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233041.613587:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761233041.613986:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233041.613987:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761233041.614261:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233041.614263:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000100:00080000:0.0:1761233046.733026:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233046.733035:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233046.733037:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233046.733039:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233046.733041:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233046.733045:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233046.733050:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5515 00000100:00080000:0.0:1761233046.733052:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233046.733054:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733056:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233046.733058:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733059:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233046.733062:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233046.733063:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233046.733065:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233046.733068:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233046.733069:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233046.733071:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233046.733073:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733074:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233046.733075:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733076:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233046.733078:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233046.733081:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733082:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233046.733092:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233046.733095:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233046.733097:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733098:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233046.733102:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233046.733111:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233046.733124:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233046.733128:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233046.733129:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733131:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233046.733133:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233046.733136:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233046.733142:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233046.733147:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233046.733152:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245189312/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233046.733159:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233046.733161:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233046.733163:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245189312/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233046.733178:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733179:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233046.733180:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233046.733183:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233046.733183:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233046.733184:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233046.733185:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233046.733187:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233046.733188:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5515 00000100:00080000:0.0:1761233046.733189:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233046.733191:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733192:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233046.733193:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733194:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233046.733195:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233046.733196:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233046.733196:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233046.733198:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233046.733199:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233046.733201:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233046.733202:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733203:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233046.733204:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733205:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233046.733206:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233046.733207:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733209:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233046.733209:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233046.733214:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233046.733215:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733216:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233046.733219:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233046.733221:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733222:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233046.733225:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233046.733226:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233046.733227:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233046.733228:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733229:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233046.733230:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233046.733231:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233046.733233:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233046.733236:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233046.733239:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245189376/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233046.733245:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233046.733247:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233046.733252:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245189376/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233046.733283:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733286:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233046.733287:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233046.733290:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233046.733291:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233046.733292:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233046.733294:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233046.733296:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233046.733299:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5515 00000100:00080000:0.0:1761233046.733301:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233046.733305:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733309:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233046.733311:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733313:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233046.733315:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233046.733316:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233046.733317:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233046.733319:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233046.733321:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233046.733325:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233046.733328:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733330:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233046.733331:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733331:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233046.733333:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233046.733335:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233046.733338:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233046.733341:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233046.733344:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233046.733346:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733348:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233046.733350:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233046.733352:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733353:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233046.733356:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233046.733359:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233046.733360:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233046.733361:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733362:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233046.733363:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233046.733365:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233046.733371:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233046.733374:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233046.733378:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245189440/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233046.733385:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233046.733387:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233046.733392:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245189440/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233046.733404:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233046.733406:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233046.733409:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5525) 00000800:00000200:1.1:1761233046.733508:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233046.733511:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761233046.733841:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233046.733867:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761233046.734146:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233046.734148:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761233051.853510:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233051.853513:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761233051.853785:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233051.853786:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761233051.853983:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233051.853984:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761233056.973797:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233056.973803:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761233056.974301:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233056.974302:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761233056.974446:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233056.974448:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761233062.093400:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233062.093403:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761233062.093602:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233062.093603:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761233062.093733:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233062.093735:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1761233067.213356:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233067.213359:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761233067.213588:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233067.213589:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761233067.213730:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233067.213731:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:0.0:1761233071.181065:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233071.181072:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233071.181077:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1761233072.333479:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233072.333481:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233072.333766:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233072.333767:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233072.333934:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233072.333936:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000020:00000001:0.0:1761233073.229098:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233073.229105:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233073.229110:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1761233077.453422:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233077.453424:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233077.453628:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233077.453647:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233077.453796:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233077.453798:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233082.573723:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233082.573729:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761233082.574232:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233082.574234:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761233082.574477:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233082.574479:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000100:00080000:0.0:1761233087.693075:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233087.693087:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233087.693090:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233087.693092:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233087.693096:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233087.693102:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233087.693110:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5556 00000100:00080000:0.0:1761233087.693113:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233087.693116:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693119:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233087.693121:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693124:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233087.693127:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233087.693129:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233087.693132:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233087.693135:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233087.693137:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233087.693140:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233087.693143:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693145:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233087.693146:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693148:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233087.693151:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233087.693155:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693157:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233087.693160:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233087.693163:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233087.693165:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693168:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233087.693170:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233087.693180:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233087.693183:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233087.693188:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233087.693189:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693192:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233087.693195:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233087.693199:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233087.693207:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233087.693214:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233087.693221:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245190848/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233087.693232:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233087.693234:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233087.693237:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245190848/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233087.693269:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693272:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233087.693273:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233087.693277:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233087.693278:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233087.693279:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233087.693281:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233087.693284:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233087.693286:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5556 00000100:00080000:0.0:1761233087.693288:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233087.693290:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693292:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233087.693293:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693295:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233087.693297:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233087.693298:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233087.693299:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233087.693301:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233087.693303:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233087.693305:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233087.693307:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693309:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233087.693311:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693313:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233087.693315:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233087.693318:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693319:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233087.693321:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233087.693327:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233087.693328:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693330:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233087.693333:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233087.693336:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693338:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233087.693341:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233087.693343:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233087.693346:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233087.693347:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693348:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233087.693349:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233087.693351:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233087.693356:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233087.693359:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233087.693362:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245190912/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233087.693370:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233087.693371:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233087.693375:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245190912/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233087.693382:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693384:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233087.693385:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233087.693388:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233087.693390:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233087.693392:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233087.693394:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233087.693399:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233087.693403:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5556 00000100:00080000:0.0:1761233087.693407:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233087.693411:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693414:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233087.693420:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693426:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233087.693431:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233087.693433:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233087.693435:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233087.693437:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233087.693442:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233087.693446:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233087.693452:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693456:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233087.693461:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693464:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233087.693490:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233087.693498:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233087.693518:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233087.693522:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233087.693525:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233087.693530:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693534:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233087.693537:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233087.693540:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693542:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233087.693549:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233087.693554:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233087.693558:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233087.693562:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693564:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233087.693566:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233087.693568:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233087.693576:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233087.693581:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233087.693589:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245190976/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233087.693607:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233087.693609:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233087.693617:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245190976/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233087.693636:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233087.693639:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233087.693646:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5566) 00000800:00000200:1.1:1761233087.693725:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233087.693893:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000800:00000200:1.1:1761233087.693927:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233087.693970:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:0.0:1761233087.694002:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000800:00000200:1.1:1761233087.694085:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233092.814150:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233092.814156:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761233092.814623:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233092.814626:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761233092.815207:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233092.815210:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233097.933474:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233097.933478:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761233097.933755:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233097.933756:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761233097.934019:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233097.934023:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1761233103.053743:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233103.053748:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1761233103.054268:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233103.054270:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1761233103.054467:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233103.054469:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000020:00000001:0.0:1761233105.997039:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233105.997042:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233105.997045:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233108.173198:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233108.173218:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233108.173221:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233108.173226:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233108.173230:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233108.173239:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233108.173249:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5577 00000100:00080000:0.0:1761233108.173254:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233108.173258:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173263:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233108.173268:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173271:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233108.173277:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233108.173280:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233108.173284:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233108.173289:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233108.173293:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233108.173296:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233108.173301:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173303:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233108.173305:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173308:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233108.173312:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233108.173317:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173320:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233108.173323:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233108.173327:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233108.173330:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173333:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233108.173335:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233108.173349:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233108.173354:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233108.173359:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233108.173360:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173364:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233108.173368:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233108.173373:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233108.173384:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233108.173393:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233108.173402:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245191616/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233108.173413:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233108.173416:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233108.173420:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245191616/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233108.173454:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173457:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233108.173459:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233108.173463:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233108.173464:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233108.173465:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233108.173467:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233108.173470:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233108.173472:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5577 00000100:00080000:0.0:1761233108.173474:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233108.173476:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173477:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233108.173479:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173480:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233108.173482:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233108.173483:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233108.173484:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233108.173486:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233108.173487:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233108.173489:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233108.173491:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173493:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233108.173495:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173496:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233108.173498:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233108.173500:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173502:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233108.173504:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233108.173511:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233108.173513:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173515:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233108.173519:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233108.173522:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173524:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233108.173527:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233108.173529:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233108.173532:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233108.173533:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173534:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233108.173535:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233108.173537:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233108.173540:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233108.173544:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233108.173547:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245191680/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233108.173555:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233108.173556:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233108.173559:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245191680/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233108.173566:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173567:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233108.173569:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233108.173572:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233108.173576:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233108.173576:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233108.173578:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233108.173582:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233108.173586:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5577 00000100:00080000:0.0:1761233108.173589:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233108.173595:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173601:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233108.173606:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173611:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233108.173616:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233108.173617:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233108.173621:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233108.173625:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233108.173631:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233108.173634:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233108.173640:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173646:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233108.173651:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173656:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233108.173661:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233108.173667:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233108.173672:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233108.173678:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233108.173682:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233108.173686:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173688:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233108.173691:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233108.173696:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173699:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233108.173704:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761233108.173710:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233108.173712:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233108.173713:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173714:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233108.173715:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233108.173717:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233108.173727:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233108.173731:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233108.173741:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245191744/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233108.173767:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233108.173770:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233108.173781:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245191744/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233108.173799:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233108.173805:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233108.173812:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5587) 00000800:00000200:1.1:1761233108.173945:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233108.173948:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761233108.174287:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233108.174290:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761233108.174571:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233108.174572:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1761233113.293753:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233113.293759:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761233113.294252:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233113.294255:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761233113.294517:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233113.294519:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761233118.414095:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233118.414105:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233118.414629:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233118.414631:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233118.414879:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233118.414881:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761233123.533510:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233123.533514:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761233123.533836:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233123.533838:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1761233123.534135:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233123.534138:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761233128.654031:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233128.654043:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233128.654813:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233128.654818:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233128.655449:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233128.655453:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233133.773910:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233133.773954:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233133.774390:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233133.774393:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233133.774656:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233133.774658:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:0.0:1761233136.717116:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233136.717138:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233136.717143:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233138.893701:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233138.893705:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1761233138.894085:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233138.894087:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1761233138.894326:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233138.894328:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1761233144.013787:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233144.013794:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1761233144.014165:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233144.014166:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1761233144.014323:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233144.014325:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000100:00080000:0.0:1761233149.133068:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233149.133080:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233149.133082:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233149.133085:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233149.133088:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233149.133093:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233149.133101:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5618 00000100:00080000:0.0:1761233149.133104:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233149.133107:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133110:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233149.133113:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133115:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233149.133119:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233149.133121:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233149.133124:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233149.133127:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233149.133129:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233149.133132:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233149.133134:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133138:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233149.133139:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133142:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233149.133144:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233149.133148:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133150:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233149.133153:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233149.133156:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233149.133159:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133161:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233149.133163:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233149.133174:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233149.133177:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233149.133182:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233149.133183:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133186:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233149.133189:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233149.133193:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233149.133201:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233149.133207:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233149.133215:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245193152/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233149.133225:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233149.133227:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233149.133232:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245193152/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233149.133258:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133260:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233149.133262:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233149.133266:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233149.133267:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233149.133269:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233149.133271:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233149.133273:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233149.133275:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5618 00000100:00080000:0.0:1761233149.133278:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233149.133280:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133282:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233149.133283:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133285:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233149.133287:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233149.133288:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233149.133289:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233149.133291:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233149.133292:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233149.133294:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233149.133296:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133298:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233149.133300:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133301:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233149.133303:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233149.133305:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133307:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233149.133309:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233149.133315:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233149.133317:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133319:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233149.133322:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233149.133325:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133327:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233149.133330:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233149.133332:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233149.133335:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233149.133336:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133339:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233149.133341:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233149.133342:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233149.133347:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233149.133352:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233149.133357:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245193216/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233149.133364:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233149.133366:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233149.133369:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245193216/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233149.133380:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133385:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233149.133387:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233149.133392:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233149.133394:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233149.133396:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233149.133398:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233149.133404:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233149.133409:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5618 00000100:00080000:0.0:1761233149.133414:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233149.133421:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133426:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233149.133430:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133435:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233149.133438:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233149.133440:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233149.133444:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233149.133448:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233149.133454:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233149.133456:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233149.133464:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133468:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233149.133475:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133482:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233149.133486:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233149.133490:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233149.133496:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233149.133500:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233149.133529:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233149.133532:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133538:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233149.133542:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233149.133545:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133547:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233149.133555:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761233149.133559:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233149.133566:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233149.133567:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133568:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233149.133571:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233149.133573:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233149.133581:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233149.133588:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233149.133599:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245193280/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233149.133620:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233149.133624:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233149.133631:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245193280/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233149.133646:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233149.133649:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233149.133654:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5628) 00000800:00000200:3.1:1761233149.133762:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233149.133766:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761233149.134200:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233149.134202:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761233149.134427:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233149.134429:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000100:00080000:0.0:1761233154.252964:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233154.252972:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233154.252974:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233154.252976:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233154.252977:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233154.252982:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233154.252988:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5623 00000100:00080000:0.0:1761233154.252990:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233154.252992:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.252994:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233154.252995:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.252997:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233154.252999:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233154.253000:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233154.253002:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233154.253005:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233154.253006:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233154.253008:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233154.253010:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253011:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233154.253012:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253013:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233154.253015:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233154.253017:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253018:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233154.253020:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233154.253022:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233154.253024:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253025:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233154.253027:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233154.253035:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ad5ae63. 00000020:00000040:0.0:1761233154.253038:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233154.253042:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233154.253043:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253045:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233154.253047:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233154.253051:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233154.253056:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233154.253061:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233154.253066:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ad5ae63 x1846784245193344/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233154.253073:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233154.253075:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233154.253077:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ad5ae63] to pc [ptlrpcd_rcv+-1] req@000000009ad5ae63 x1846784245193344/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233154.253094:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253096:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233154.253097:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233154.253099:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233154.253100:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233154.253101:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233154.253102:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233154.253103:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233154.253105:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5623 00000100:00080000:0.0:1761233154.253106:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233154.253107:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253108:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233154.253109:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253110:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233154.253111:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233154.253112:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233154.253113:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233154.253114:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233154.253115:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233154.253117:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233154.253118:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253119:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233154.253120:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253121:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233154.253122:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233154.253124:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253125:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233154.253126:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233154.253130:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233154.253131:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253132:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233154.253134:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233154.253136:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253137:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233154.253140:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000006edc786f. 00000020:00000040:0.0:1761233154.253141:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233154.253142:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233154.253143:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253144:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233154.253145:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233154.253146:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233154.253148:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233154.253150:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233154.253154:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006edc786f x1846784245193408/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233154.253159:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233154.253161:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233154.253165:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000006edc786f] to pc [ptlrpcd_rcv+-1] req@000000006edc786f x1846784245193408/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233154.253173:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253175:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233154.253176:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233154.253180:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233154.253181:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233154.253182:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233154.253183:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233154.253187:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233154.253190:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5623 00000100:00080000:0.0:1761233154.253192:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233154.253196:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253200:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233154.253201:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253203:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233154.253205:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233154.253207:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233154.253209:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233154.253211:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233154.253213:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233154.253215:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233154.253220:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253223:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233154.253226:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253229:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233154.253230:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233154.253231:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233154.253234:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233154.253235:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233154.253237:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233154.253239:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253240:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233154.253243:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233154.253245:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253245:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233154.253249:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761233154.253253:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233154.253256:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233154.253257:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253258:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233154.253259:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233154.253259:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233154.253262:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233154.253266:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233154.253269:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245193472/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233154.253274:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233154.253276:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233154.253278:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245193472/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233154.253283:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233154.253286:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233154.253289:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5633) 00000800:00000200:3.1:1761233154.253376:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233154.253496:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000400:00000200:0.0:1761233154.253548:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:0.0:1761233154.253567:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000800:00000200:3.1:1761233154.253579:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233154.253716:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00080000:0.0:1761233159.373072:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233159.373081:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233159.373082:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233159.373085:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233159.373087:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233159.373091:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233159.373096:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5628 00000100:00080000:0.0:1761233159.373099:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233159.373102:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373107:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233159.373108:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373110:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233159.373113:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233159.373115:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233159.373117:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233159.373120:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233159.373121:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233159.373123:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233159.373125:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373127:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233159.373128:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373129:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233159.373131:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233159.373133:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373135:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233159.373137:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233159.373140:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233159.373141:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373143:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233159.373144:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233159.373149:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233159.373152:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233159.373156:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233159.373158:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373161:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233159.373163:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233159.373176:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233159.373182:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233159.373188:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233159.373194:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245193536/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233159.373206:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233159.373208:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233159.373211:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245193536/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233159.373239:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373241:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233159.373243:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233159.373245:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233159.373246:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233159.373247:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233159.373248:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233159.373249:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233159.373250:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5628 00000100:00080000:0.0:1761233159.373251:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233159.373253:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373254:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233159.373255:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373256:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233159.373258:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233159.373258:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233159.373259:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233159.373260:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233159.373261:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233159.373263:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233159.373265:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373266:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233159.373268:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373269:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233159.373271:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233159.373273:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373275:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233159.373277:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233159.373283:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233159.373285:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373287:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233159.373290:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233159.373294:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373295:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233159.373298:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233159.373300:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233159.373303:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233159.373304:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373306:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233159.373307:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233159.373309:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233159.373314:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233159.373319:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233159.373327:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245193600/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233159.373360:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233159.373362:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233159.373364:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245193600/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233159.373371:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373372:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233159.373374:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233159.373376:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233159.373377:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233159.373380:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233159.373381:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233159.373386:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233159.373390:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5628 00000100:00080000:0.0:1761233159.373393:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233159.373398:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373401:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233159.373405:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373409:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233159.373414:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233159.373416:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233159.373418:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233159.373420:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233159.373425:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233159.373428:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233159.373433:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373436:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233159.373440:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373443:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233159.373446:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233159.373452:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233159.373466:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233159.373468:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233159.373472:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233159.373476:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373477:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233159.373480:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233159.373484:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373485:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233159.373489:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761233159.373492:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233159.373495:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233159.373499:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373502:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233159.373505:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233159.373508:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233159.373513:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233159.373518:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233159.373526:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245193664/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233159.373537:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233159.373539:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233159.373544:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245193664/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233159.373558:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233159.373561:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233159.373565:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5638) 00000800:00000200:3.1:1761233159.373647:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233159.373649:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233159.374030:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233159.374032:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233159.374381:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233159.374382:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233164.493505:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233164.493508:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233164.493794:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233164.493795:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233164.494033:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233164.494035:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233169.613538:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233169.613543:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1761233169.613945:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233169.613948:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1761233169.614243:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233169.614245:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1761233174.733408:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233174.733410:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1761233174.733616:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233174.733617:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1761233174.733745:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233174.733758:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761233179.853831:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233179.853836:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761233179.854214:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233179.854215:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1761233179.854349:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233179.854350:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1761233184.973834:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233184.973839:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761233184.974322:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233184.974324:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761233184.974569:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233184.974571:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761233190.093575:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233190.093580:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233190.094232:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233190.094235:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233190.094532:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233190.094534:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233195.213384:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233195.213386:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233195.213560:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233195.213562:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233195.213817:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233195.213819:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233200.333683:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233200.333687:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233200.334132:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233200.334135:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233200.334457:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233200.334481:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761233205.453433:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233205.453435:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233205.453675:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233205.453677:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233205.453814:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233205.453815:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233210.573420:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233210.573423:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233210.573701:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233210.573702:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233210.573865:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233210.573867:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761233215.693446:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233215.693449:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761233215.693711:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233215.693712:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761233215.693864:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233215.693865:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761233220.813361:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233220.813491:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233220.813592:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000020:00000001:0.0:1761233224.781001:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233224.781005:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233224.781008:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1761233225.933449:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233225.933454:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761233225.933773:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233225.933775:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761233225.934043:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233225.934046:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1761233231.053914:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233231.053923:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761233231.054337:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233231.054340:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761233231.054620:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233231.054622:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000020:00000001:0.0:1761233235.021051:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233235.021056:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233235.021061:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233236.173879:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233236.173883:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1761233236.174304:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233236.174306:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1761233236.174562:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233236.174565:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000020:00000001:0.0:1761233237.069055:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233237.069059:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233237.069063:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233239.117081:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233239.117103:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233239.117108:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233241.165011:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233241.165015:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233241.165018:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233241.293136:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233241.293154:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233241.293156:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233241.293160:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233241.293164:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233241.293185:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233241.293205:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5710 00000100:00080000:0.0:1761233241.293211:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233241.293217:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293222:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233241.293226:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293229:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233241.293236:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233241.293239:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233241.293244:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233241.293250:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233241.293254:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233241.293259:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233241.293264:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293268:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233241.293271:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293274:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233241.293278:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233241.293285:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293289:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233241.293294:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233241.293300:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233241.293304:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293308:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233241.293312:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233241.293329:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233241.293336:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233241.293344:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233241.293346:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293352:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233241.293357:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233241.293363:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233241.293379:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233241.293389:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233241.293400:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245196608/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233241.293412:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233241.293415:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233241.293419:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245196608/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233241.293451:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293454:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233241.293457:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233241.293461:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233241.293462:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233241.293463:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233241.293465:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233241.293468:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233241.293471:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5710 00000100:00080000:0.0:1761233241.293473:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233241.293476:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293478:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233241.293480:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293481:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233241.293484:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233241.293485:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233241.293486:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233241.293489:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233241.293490:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233241.293492:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233241.293495:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293497:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233241.293498:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293501:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233241.293504:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233241.293506:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293509:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233241.293511:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233241.293519:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233241.293521:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293523:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233241.293526:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233241.293530:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293533:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233241.293537:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233241.293540:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233241.293543:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233241.293544:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293547:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233241.293548:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233241.293551:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233241.293556:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233241.293560:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233241.293565:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245196672/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233241.293573:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233241.293575:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233241.293583:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245196672/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233241.293596:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293599:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233241.293601:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233241.293606:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233241.293608:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233241.293611:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233241.293614:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233241.293621:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233241.293627:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5710 00000100:00080000:0.0:1761233241.293632:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233241.293639:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293643:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233241.293647:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293653:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233241.293656:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233241.293659:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233241.293662:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233241.293665:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233241.293670:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233241.293675:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233241.293682:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293687:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233241.293691:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293693:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233241.293695:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233241.293699:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233241.293705:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233241.293709:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233241.293713:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233241.293715:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293717:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233241.293718:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233241.293721:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293722:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233241.293726:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233241.293728:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233241.293730:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233241.293732:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293746:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233241.293748:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233241.293751:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233241.293758:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233241.293763:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233241.293772:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245196736/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233241.293799:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233241.293801:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233241.293808:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245196736/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233241.293836:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233241.293857:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233241.293865:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5720) 00000800:00000200:3.1:1761233241.294026:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233241.294030:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1761233241.294465:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233241.294468:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1761233241.294704:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233241.294706:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000020:00000001:0.0:1761233243.213070:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233243.213076:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233243.213080:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233245.261081:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233245.261085:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233245.261089:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233246.413560:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233246.413562:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1761233246.413790:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233246.413791:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1761233246.413961:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233246.413995:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000020:00000001:0.0:1761233247.309152:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233247.309156:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233247.309160:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233249.357091:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233249.357095:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233249.357099:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233251.405056:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233251.405061:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233251.405066:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1761233251.533497:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233251.533500:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761233251.533747:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233251.533749:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1761233251.533924:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233251.533925:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000020:00000001:0.0:1761233253.453126:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233253.453131:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233253.453135:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233255.501085:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233255.501088:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233255.501092:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233256.653426:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233256.653430:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1761233256.653704:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233256.653705:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1761233256.653882:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233256.653883:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000020:00000001:0.0:1761233257.549089:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233257.549091:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233257.549094:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233261.773482:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233261.773484:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1761233261.773688:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233261.773689:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1761233261.773839:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233261.773841:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1761233266.893996:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233266.894001:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761233266.894604:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233266.894606:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1761233266.894935:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233266.894938:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:0.0:1761233268.813021:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233268.813024:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233268.813027:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233270.861007:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233270.861011:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233270.861014:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1761233272.013679:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233272.013682:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233272.014030:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233272.014032:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233272.014272:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233272.014275:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000020:00000001:0.0:1761233272.909004:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233272.909008:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233272.909010:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233274.957069:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233274.957075:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233274.957080:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233277.005120:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233277.005124:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233277.005128:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1761233277.133479:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233277.133482:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233277.133734:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233277.133735:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233277.133904:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233277.133906:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000020:00000001:0.0:1761233279.053060:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233279.053064:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233279.053068:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233281.101035:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233281.101039:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233281.101041:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1761233282.253765:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233282.253769:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761233282.254200:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233282.254203:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1761233282.254602:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233282.254605:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000020:00000001:0.0:1761233283.149100:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233283.149104:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233283.149107:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233287.373511:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233287.373658:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233287.373670:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000400:00000200:0.0:1761233287.373750:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000800:00000200:3.1:1761233287.373766:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233287.373793:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000400:00000100:0.0:1761233290.445159:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:0.0:1761233290.445202:0:88985:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1761233290.445210:0:88985:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:0.0:1761233290.445220:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233290.445225:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000171ed136 (tot 1676778). 00000400:00000200:0.0:1761233290.445231:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233290.445235:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233290.445240:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:0.0:1761233290.445242:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:0.0:1761233290.445248:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233290.445256:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000003630282c x1846784245198336/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233312 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233290.445328:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233290.445334:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003630282c x1846784245198336/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233312 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233290.445344:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233290.445347:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233290.445362:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233290.445367:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000eb8d1c51 (tot 1676530). 00000400:00000200:0.0:1761233290.445373:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233290.445383:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233290.445390:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000046716e3e 00000400:00000010:0.0:1761233290.445393:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000046716e3e. 00000100:00000001:0.0:1761233290.445397:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233290.445405:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000f4f489ba x1846784245198400/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233312 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233290.445423:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233290.445429:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f4f489ba x1846784245198400/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233312 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233290.445446:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233290.445451:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233290.445463:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233290.445466:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006f6738d0 (tot 1676282). 00000400:00000200:0.0:1761233290.445471:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233290.445484:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233290.445492:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000faee444f 00000400:00000010:0.0:1761233290.445496:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000faee444f. 00000100:00000001:0.0:1761233290.445501:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233290.445517:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000c3b47c4a x1846784245198464/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233312 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233290.445536:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233290.445543:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000c3b47c4a x1846784245198464/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233312 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233290.445558:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233290.445562:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000200:3.1:1761233292.493638:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233292.493644:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233292.494174:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233292.494177:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233292.494597:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233292.494599:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233297.613609:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233297.613612:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233297.613960:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233297.613979:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233297.614164:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233297.614165:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761233302.733614:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233302.733617:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761233302.733886:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233302.733888:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761233302.734188:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233302.734189:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000020:00000001:0.0:1761233303.629070:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233303.629092:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233303.629108:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233307.853027:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233307.853042:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233307.853044:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233307.853048:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233307.853051:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233307.853057:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233307.853065:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5776 00000100:00080000:0.0:1761233307.853069:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233307.853071:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853074:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233307.853076:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853078:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233307.853082:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233307.853085:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233307.853088:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233307.853091:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233307.853093:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233307.853096:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233307.853098:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853100:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233307.853103:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853104:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233307.853107:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233307.853110:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853113:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233307.853116:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233307.853119:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233307.853121:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853123:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233307.853126:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233307.853136:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233307.853139:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233307.853144:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233307.853145:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853149:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233307.853152:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233307.853157:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233307.853167:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233307.853175:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233307.853183:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245199104/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233307.853192:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233307.853195:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233307.853199:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245199104/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233307.853220:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853222:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233307.853225:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233307.853229:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233307.853231:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233307.853232:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233307.853240:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233307.853244:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233307.853247:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5776 00000100:00080000:0.0:1761233307.853251:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233307.853271:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853274:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233307.853277:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853279:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233307.853283:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233307.853284:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233307.853286:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233307.853289:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233307.853293:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233307.853295:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233307.853299:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853302:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233307.853304:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853307:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233307.853309:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233307.853313:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853332:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233307.853335:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233307.853343:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233307.853346:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853349:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233307.853353:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233307.853359:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853361:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233307.853366:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233307.853370:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233307.853374:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233307.853376:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853379:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233307.853382:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233307.853385:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233307.853392:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233307.853397:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233307.853402:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245199168/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233307.853412:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233307.853414:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233307.853418:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245199168/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233307.853431:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853434:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233307.853437:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233307.853442:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233307.853444:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233307.853446:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233307.853448:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233307.853452:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233307.853456:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5776 00000100:00080000:0.0:1761233307.853459:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233307.853462:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853465:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233307.853468:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853471:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233307.853474:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233307.853476:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233307.853478:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233307.853480:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233307.853482:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233307.853485:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233307.853489:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853493:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233307.853496:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853498:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233307.853501:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233307.853505:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233307.853508:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233307.853510:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233307.853514:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233307.853533:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853536:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233307.853539:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233307.853543:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853545:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233307.853550:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233307.853553:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233307.853557:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233307.853559:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853562:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233307.853564:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233307.853566:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233307.853572:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233307.853575:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233307.853581:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245199232/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233307.853593:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233307.853595:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233307.853599:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245199232/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:1.1:1761233307.853603:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233307.853608:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000100:00000001:0.0:1761233307.853611:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233307.853614:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233307.853636:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5786) 00000800:00000200:1.1:1761233307.853988:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233307.853990:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761233307.854203:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233307.854205:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000020:00000001:0.0:1761233309.773057:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233309.773060:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233309.773064:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233311.821015:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233311.821019:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233311.821021:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1761233312.973980:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233312.973986:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761233312.974309:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233312.974311:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761233312.974725:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233312.974727:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1761233318.093955:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233318.093959:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761233318.094531:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233318.094534:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1761233318.094806:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233318.094809:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000100:00000040:1.0:1761233323.213554:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233323.213558:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233323.213567:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233323.213573:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233323.213576:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233323.213580:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233323.213583:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233323.213585:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233323.213588:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233323.213590:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233323.213592:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233323.213595:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233323.213599:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233323.213600:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233323.213604:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761233323.213609:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233323.213615:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233323.213619:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233323.213625:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233323.213628:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233323.213631:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233323.213653:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233323.213659:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233323.213686:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245199808/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233323.213722:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233323.213725:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233323.213757:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245199808/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233323.213790:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233323.213792:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233323.213828:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5802) 00000800:00000200:3.1:1761233323.213844:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233323.213848:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233323.214155:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233323.214156:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233323.214309:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233323.214310:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1761233325.133114:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233325.133117:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233325.133121:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233327.181198:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233327.181202:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233327.181206:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233328.333015:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233328.333031:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233328.333034:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233328.333037:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233328.333041:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233328.333048:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233328.333058:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5797 00000100:00080000:1.0:1761233328.333062:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233328.333064:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333067:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233328.333069:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333071:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233328.333075:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233328.333076:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233328.333079:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233328.333082:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233328.333085:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233328.333087:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233328.333089:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333090:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233328.333091:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333092:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233328.333094:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233328.333096:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333098:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233328.333101:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233328.333104:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233328.333107:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333110:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233328.333112:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233328.333122:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761233328.333125:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233328.333130:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233328.333131:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333135:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233328.333138:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233328.333142:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233328.333150:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233328.333157:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233328.333163:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245199872/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233328.333172:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233328.333174:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233328.333179:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245199872/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233328.333206:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333208:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233328.333211:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233328.333214:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233328.333215:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233328.333217:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233328.333218:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233328.333221:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233328.333224:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5797 00000100:00080000:1.0:1761233328.333226:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233328.333229:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333231:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233328.333233:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333234:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233328.333237:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233328.333238:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233328.333239:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233328.333241:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233328.333243:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233328.333246:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233328.333250:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333252:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233328.333253:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333255:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233328.333257:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233328.333259:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333262:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233328.333265:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233328.333273:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233328.333276:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333278:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233328.333282:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233328.333285:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333287:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233328.333292:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000020:00000040:1.0:1761233328.333295:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233328.333300:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233328.333302:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333307:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233328.333311:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233328.333313:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233328.333318:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233328.333332:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233328.333346:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245199936/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233328.333366:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233328.333370:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233328.333378:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245199936/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233328.333403:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333406:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233328.333414:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233328.333424:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233328.333428:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233328.333431:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233328.333432:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233328.333435:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233328.333440:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5797 00000100:00080000:1.0:1761233328.333444:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233328.333448:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333456:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233328.333459:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333466:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233328.333471:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233328.333480:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233328.333483:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233328.333485:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233328.333489:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233328.333495:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233328.333503:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333506:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233328.333513:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333518:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233328.333526:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233328.333532:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233328.333535:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233328.333542:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233328.333546:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233328.333549:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333550:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233328.333552:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233328.333554:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333556:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233328.333560:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761233328.333570:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233328.333574:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233328.333578:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333584:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233328.333589:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233328.333593:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233328.333599:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233328.333607:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233328.333616:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245200000/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233328.333623:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233328.333623:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233328.333628:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245200000/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233328.333653:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233328.333658:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233328.333663:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5807) 00000800:00000200:3.1:1761233328.333683:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233328.333702:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233328.333998:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233328.334000:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233328.334196:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233328.334198:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1761233329.229091:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233329.229094:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233329.229097:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233331.277027:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233331.277030:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233331.277033:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233333.325212:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233333.325221:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233333.325231:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233333.453063:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233333.453081:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233333.453083:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233333.453088:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233333.453092:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233333.453101:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233333.453112:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5802 00000100:00080000:1.0:1761233333.453117:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233333.453121:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453125:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233333.453128:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453131:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233333.453136:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233333.453140:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233333.453144:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233333.453149:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233333.453152:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233333.453155:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233333.453158:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453160:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233333.453162:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453165:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233333.453168:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233333.453173:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453176:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233333.453180:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233333.453184:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233333.453188:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453191:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233333.453194:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233333.453210:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761233333.453215:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233333.453221:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233333.453223:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453228:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233333.453233:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233333.453256:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233333.453272:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233333.453282:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233333.453293:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245200064/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233333.453314:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233333.453317:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233333.453323:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245200064/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233333.453352:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453355:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233333.453357:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233333.453362:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233333.453363:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233333.453365:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233333.453366:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233333.453369:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233333.453372:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5802 00000100:00080000:1.0:1761233333.453374:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233333.453377:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453379:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233333.453381:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453383:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233333.453385:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233333.453386:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233333.453387:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233333.453390:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233333.453392:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233333.453396:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233333.453399:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453401:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233333.453403:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453405:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233333.453407:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233333.453410:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453412:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233333.453414:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233333.453422:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233333.453424:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453426:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233333.453430:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233333.453434:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453436:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233333.453441:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761233333.453444:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233333.453447:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233333.453448:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453451:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233333.453452:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233333.453454:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233333.453459:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233333.453463:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233333.453469:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245200128/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233333.453478:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233333.453479:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233333.453483:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245200128/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233333.453492:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453496:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233333.453500:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233333.453521:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233333.453524:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233333.453526:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233333.453528:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233333.453531:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233333.453536:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5802 00000100:00080000:1.0:1761233333.453538:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233333.453542:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453546:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233333.453549:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453553:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233333.453557:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233333.453558:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233333.453560:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233333.453562:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233333.453566:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233333.453569:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233333.453572:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453576:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233333.453579:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453582:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233333.453584:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233333.453586:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233333.453589:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233333.453590:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233333.453593:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233333.453598:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453600:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233333.453602:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233333.453626:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453628:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233333.453631:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761233333.453633:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233333.453635:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233333.453636:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453638:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233333.453639:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233333.453641:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233333.453644:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233333.453649:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233333.453657:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245200192/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233333.453665:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233333.453668:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233333.453675:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245200192/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233333.453684:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233333.453688:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233333.453693:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5812) 00000800:00000200:3.1:1761233333.453941:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233333.453948:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233333.454437:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233333.454439:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233333.454775:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233333.454777:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233335.373048:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233335.373051:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233335.373053:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233337.421268:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233337.421275:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233337.421280:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1761233338.573325:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233338.573333:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233338.573337:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233338.573351:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573391:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233338.573395:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233338.573402:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233338.573407:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233338.573421:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245200256:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233338.573431:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233338.573437:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233338.573439:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233338.573448:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233338.573458:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000c6b52d7d. 02000000:00000001:1.0:1761233338.573461:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233338.573470:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233338.573477:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233338.573484:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245200256, portal 25 00000100:00000001:1.0:1761233338.573490:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233338.573493:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091124736 : -104717618426880 : ffffa0c2878d4c00) 00000100:00000040:1.0:1761233338.573504:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233363 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573522:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233338.573541:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233338.573543:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233338.573545:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245200256, offset 0 00000400:00000200:1.0:1761233338.573553:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233338.573570:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233338.573581:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233338.573589:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233338.573598:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233338.573607:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11346:11346:256:4294967295] 192.168.201.124@tcp LPNI seq info [11346:11346:8:4294967295] 00000400:00000200:1.0:1761233338.573628:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233338.573637:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233338.573645:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233338.573654:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233338.573656:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233338.573661:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233338.573663:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1761233338.573669:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233338.573671:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233338.573677:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233338.573680:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233338.573683:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233338.573685:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233338.573688:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233338.573691:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233363 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573698:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233338.573700:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233363 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573704:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.573705:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233338.573709:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.573710:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233338.573714:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.573716:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233338.573717:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233338/real 1761233338] req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233338.573724:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233338.573725:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233338.573726:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233338.573727:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233338.573728:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233338.573729:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233338.573730:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233338.573735:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573738:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233338.573739:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.573741:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233338.573742:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233338.573744:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573752:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233338.573754:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233338.573759:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233338.573760:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.573761:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233338.573762:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.573764:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233338.573765:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233338.573767:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233338.573769:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233338.573770:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233338.573772:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233338.573776:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245200256:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233338.573779:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.573780:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233338.573781:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f61de4c8 x1846784245200256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233338.573785:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233338.573787:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233338.573789:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:1.0:1761233338.573791:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233338.573792:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233338.573793:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:1.0:1761233338.573795:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233338.573797:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000be06bfbf. 02000000:00000001:1.0:1761233338.573800:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233338.573801:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233338.573801:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f61de4c8. 00000100:00000001:1.0:1761233338.573806:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233338.573806:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.573808:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.573814:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233338.573815:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.573817:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233338.573818:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233338.573819:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233338.573821:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573826:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233338.573827:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233338.573828:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233338.573829:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233338.573831:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245200320:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233338.573833:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233338.573833:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233338.573834:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233338.573835:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233338.573837:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000c6b52d7d. 02000000:00000001:1.0:1761233338.573838:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233338.573840:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233338.573841:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233338.573864:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245200320, portal 10 00000100:00000001:1.0:1761233338.573865:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233338.573866:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091128384 : -104717618423232 : ffffa0c2878d5a40) 00000100:00000040:1.0:1761233338.573870:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233363 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573874:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233338.573876:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233338.573877:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233338.573878:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245200320, offset 0 00000400:00000200:1.0:1761233338.573881:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233338.573884:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233338.573887:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233338.573888:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233338.573890:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233338.573891:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11347:11347:256:4294967295] 192.168.201.124@tcp LPNI seq info [11347:11347:8:4294967295] 00000400:00000200:1.0:1761233338.573896:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233338.573899:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233338.573900:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233338.573903:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233338.573904:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233338.573907:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233338.573908:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1761233338.573910:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233338.573912:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233338.573913:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233338.573915:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233338.573917:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233338.573918:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233338.573919:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233338.573921:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233363 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573925:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233338.573926:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233363 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573930:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.573931:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233338.573932:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.573933:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233338.573934:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.573935:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233338.573936:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233338/real 1761233338] req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233338.573941:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233338.573942:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233338.573943:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233338.573944:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233338.573944:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233338.573945:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233338.573947:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233338.573950:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573954:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233338.573954:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.573955:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233338.573956:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233338.573957:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573961:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233338.573962:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233338.573966:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233338.573967:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.573968:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233338.573969:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.573970:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233338.573970:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233338.573972:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233338.573973:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233338.573974:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233338.573976:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233338.573980:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245200320:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233338.573984:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233338.573986:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.573990:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233338.573991:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233338.573992:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233338.573993:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233338.573994:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245200384:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233338.573996:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233338.573997:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233338.573997:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233338.573999:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233338.574000:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f43b35ab. 02000000:00000001:1.0:1761233338.574001:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233338.574003:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233338.574004:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233338.574006:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245200384, portal 4 00000100:00000001:1.0:1761233338.574007:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233338.574007:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091110144 : -104717618441472 : ffffa0c2878d1300) 00000100:00000040:1.0:1761233338.574010:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233363 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.574013:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233338.574015:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233338.574016:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233338.574017:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245200384, offset 0 00000400:00000200:1.0:1761233338.574019:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233338.574021:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233338.574023:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233338.574024:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233338.574025:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233338.574027:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11348:11348:256:4294967295] 192.168.201.124@tcp LPNI seq info [11348:11348:8:4294967295] 00000400:00000200:1.0:1761233338.574032:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233338.574035:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233338.574036:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233338.574038:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233338.574039:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1761233338.574042:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233338.574043:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1761233338.574045:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233338.574046:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233338.574047:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233338.574049:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233338.574050:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233338.574051:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233338.574052:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233338.574053:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233363 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.574057:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233338.574058:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233363 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.574062:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.574062:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233338.574064:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.574064:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233338.574065:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.574066:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233338.574067:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233338/real 1761233338] req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233338.574071:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233338.574072:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233338.574073:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233338.574074:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233338.574075:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233338.574076:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233338.574077:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233338.574084:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.574088:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233338.574089:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.574090:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233338.574091:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233338.574092:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233338.574095:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233338.574097:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233338.574100:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233338.574101:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.574102:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233338.574102:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.574103:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233338.574104:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233338.574105:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233338.574106:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233338.574107:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233338.574109:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233338.574112:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245200384:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233338.574114:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.574115:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233338.574116:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002431276f x1846784245200320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233338.574120:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233338.574121:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233338.574122:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:1.0:1761233338.574123:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233338.574124:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233338.574125:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:1.0:1761233338.574126:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233338.574127:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000296782d7. 02000000:00000001:1.0:1761233338.574129:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233338.574129:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233338.574130:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002431276f. 00000100:00000001:1.0:1761233338.574132:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233338.574132:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.574133:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233338.574134:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000108ab0a5 x1846784245200384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233363 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233338.574138:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233338.574139:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233338.574140:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f43b35ab. 02000000:00000001:1.0:1761233338.574141:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233338.574142:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233338.574142:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:1.0:1761233338.574143:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233338.574143:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062959873. 02000000:00000001:1.0:1761233338.574145:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233338.574146:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233338.574146:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000108ab0a5. 00000100:00000001:1.0:1761233338.574148:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233338.574149:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.574150:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233338.574152:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233338.574152:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233338.574153:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233338.574154:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233339.469067:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233339.469071:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233339.469074:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233341.517034:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233341.517038:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233341.517041:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233343.565105:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233343.565108:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233343.565111:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1761233343.693217:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233343.693222:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233343.693226:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233343.693236:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693257:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233343.693259:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233343.693265:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233343.693269:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233343.693278:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245200448:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233343.693288:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233343.693294:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233343.693298:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233343.693304:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233343.693311:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f43b35ab. 02000000:00000001:1.0:1761233343.693316:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233343.693327:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233343.693337:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233343.693346:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245200448, portal 25 00000100:00000001:1.0:1761233343.693351:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233343.693354:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091110144 : -104717618441472 : ffffa0c2878d1300) 00000100:00000040:1.0:1761233343.693364:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233368 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693376:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233343.693381:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233343.693384:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233343.693387:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245200448, offset 0 00000400:00000200:1.0:1761233343.693396:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233343.693410:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233343.693419:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233343.693423:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233343.693426:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233343.693430:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11349:11349:256:4294967295] 192.168.201.124@tcp LPNI seq info [11349:11349:8:4294967295] 00000400:00000200:1.0:1761233343.693441:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233343.693465:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233343.693470:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233343.693477:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233343.693480:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233343.693486:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233343.693489:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1761233343.693493:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233343.693496:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233343.693502:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233343.693507:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233343.693512:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233343.693514:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233343.693518:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233343.693521:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233368 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693530:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233343.693533:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233368 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693539:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.693541:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233343.693546:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.693548:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233343.693551:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.693553:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233343.693556:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233343/real 1761233343] req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233343.693565:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233343.693567:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233343.693568:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233343.693570:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233343.693572:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233343.693574:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233343.693576:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233343.693582:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693588:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233343.693590:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.693592:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233343.693594:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233343.693597:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693604:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233343.693608:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233343.693616:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233343.693618:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.693620:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233343.693621:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.693623:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233343.693625:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233343.693627:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233343.693630:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233343.693633:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233343.693637:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233343.693645:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245200448:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233343.693650:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.693653:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233343.693655:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000108ab0a5 x1846784245200448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233343.693662:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233343.693664:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233343.693666:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f43b35ab. 02000000:00000001:1.0:1761233343.693669:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233343.693671:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233343.693672:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:1.0:1761233343.693674:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233343.693676:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074003a94. 02000000:00000001:1.0:1761233343.693680:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233343.693682:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233343.693683:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000108ab0a5. 00000100:00000001:1.0:1761233343.693687:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233343.693688:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.693690:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.693698:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233343.693700:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.693702:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233343.693703:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233343.693704:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233343.693707:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693713:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233343.693714:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233343.693716:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233343.693718:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233343.693720:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245200512:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233343.693723:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233343.693724:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233343.693726:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233343.693727:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233343.693730:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f43b35ab. 02000000:00000001:1.0:1761233343.693731:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233343.693734:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233343.693737:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233343.693740:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245200512, portal 10 00000100:00000001:1.0:1761233343.693741:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233343.693742:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091128384 : -104717618423232 : ffffa0c2878d5a40) 00000100:00000040:1.0:1761233343.693747:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233368 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693753:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233343.693756:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233343.693758:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233343.693760:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245200512, offset 0 00000400:00000200:1.0:1761233343.693765:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233343.693771:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233343.693775:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233343.693777:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233343.693780:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233343.693783:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11350:11350:256:4294967295] 192.168.201.124@tcp LPNI seq info [11350:11350:8:4294967295] 00000400:00000200:1.0:1761233343.693791:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233343.693796:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233343.693799:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233343.693804:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233343.693806:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233343.693810:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233343.693813:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1761233343.693817:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233343.693818:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233343.693821:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233343.693824:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233343.693827:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233343.693829:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233343.693830:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233343.693833:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233368 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693841:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233343.693874:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233368 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693881:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.693883:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233343.693887:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.693888:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233343.693890:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.693892:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233343.693895:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233343/real 1761233343] req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233343.693903:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233343.693904:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233343.693906:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233343.693908:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233343.693909:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233343.693911:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233343.693913:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233343.693921:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693927:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233343.693928:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.693930:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233343.693932:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233343.693934:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693941:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233343.693943:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233343.693950:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233343.693952:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.693953:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233343.693955:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.693956:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233343.693958:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233343.693960:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233343.693962:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233343.693964:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233343.693966:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233343.693973:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245200512:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233343.693977:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233343.693979:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.693985:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233343.693987:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233343.693989:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233343.693991:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233343.693994:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245200576:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233343.693997:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233343.693999:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233343.694000:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233343.694001:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233343.694004:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000c6b52d7d. 02000000:00000001:1.0:1761233343.694006:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233343.694009:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233343.694012:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233343.694015:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245200576, portal 4 00000100:00000001:1.0:1761233343.694017:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233343.694018:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091124736 : -104717618426880 : ffffa0c2878d4c00) 00000100:00000040:1.0:1761233343.694022:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233368 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.694029:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233343.694032:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233343.694034:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233343.694036:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245200576, offset 0 00000400:00000200:1.0:1761233343.694041:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233343.694046:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233343.694049:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233343.694052:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233343.694054:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233343.694057:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11351:11351:256:4294967295] 192.168.201.124@tcp LPNI seq info [11351:11351:8:4294967295] 00000400:00000200:1.0:1761233343.694064:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233343.694069:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233343.694072:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233343.694076:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233343.694079:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1761233343.694083:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233343.694085:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1761233343.694089:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233343.694091:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233343.694093:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233343.694096:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233343.694099:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233343.694100:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233343.694102:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233343.694105:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233368 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.694111:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233343.694113:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233368 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.694119:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.694120:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233343.694123:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.694124:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233343.694126:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.694128:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233343.694130:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233343/real 1761233343] req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233343.694137:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233343.694138:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233343.694139:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233343.694141:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233343.694142:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233343.694143:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233343.694145:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233343.694152:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.694158:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233343.694159:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.694161:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233343.694162:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233343.694165:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233343.694172:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233343.694175:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233343.694181:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233343.694183:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.694184:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233343.694185:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.694186:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233343.694188:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233343.694189:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233343.694192:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233343.694194:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233343.694197:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233343.694204:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245200576:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233343.694208:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.694210:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233343.694212:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002431276f x1846784245200512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233343.694219:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233343.694220:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233343.694222:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f43b35ab. 02000000:00000001:1.0:1761233343.694225:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233343.694226:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233343.694228:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:1.0:1761233343.694230:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233343.694231:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074a27ae0. 02000000:00000001:1.0:1761233343.694233:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233343.694234:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233343.694235:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002431276f. 00000100:00000001:1.0:1761233343.694238:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233343.694239:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.694240:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233343.694243:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f61de4c8 x1846784245200576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233368 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233343.694250:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233343.694251:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233343.694252:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:1.0:1761233343.694254:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233343.694255:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233343.694256:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:1.0:1761233343.694258:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233343.694259:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000009baf651a. 02000000:00000001:1.0:1761233343.694261:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233343.694262:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233343.694263:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f61de4c8. 00000100:00000001:1.0:1761233343.694266:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233343.694267:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.694269:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233343.694273:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233343.694274:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233343.694276:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233343.694277:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233345.613064:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233345.613067:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233345.613070:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233347.661131:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233347.661137:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233347.661144:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233348.813036:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233348.813045:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233348.813047:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233348.813050:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233348.813052:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233348.813057:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233348.813063:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5817 00000100:00080000:1.0:1761233348.813066:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233348.813068:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813071:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233348.813073:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813075:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233348.813077:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233348.813079:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233348.813081:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233348.813084:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233348.813086:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233348.813087:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233348.813089:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813092:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233348.813093:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813095:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233348.813097:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233348.813100:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813102:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233348.813104:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233348.813107:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233348.813109:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813110:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233348.813112:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233348.813121:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761233348.813124:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233348.813128:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233348.813129:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813131:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233348.813133:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233348.813137:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233348.813144:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233348.813149:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233348.813155:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245200640/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233348.813164:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233348.813166:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233348.813168:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245200640/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233348.813189:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813191:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233348.813193:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233348.813196:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233348.813197:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233348.813198:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233348.813199:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233348.813202:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233348.813203:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5817 00000100:00080000:1.0:1761233348.813205:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233348.813208:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813209:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233348.813210:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813211:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233348.813213:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233348.813214:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233348.813215:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233348.813217:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233348.813218:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233348.813219:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233348.813222:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813223:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233348.813226:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813227:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233348.813228:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233348.813230:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813232:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233348.813234:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233348.813240:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233348.813241:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813243:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233348.813248:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233348.813252:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813255:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233348.813261:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761233348.813264:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233348.813268:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233348.813270:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813273:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233348.813274:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233348.813276:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233348.813281:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233348.813285:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233348.813292:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245200704/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233348.813301:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233348.813303:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233348.813315:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245200704/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233348.813330:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813333:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233348.813336:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233348.813339:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233348.813342:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233348.813344:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233348.813347:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233348.813349:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233348.813351:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5817 00000100:00080000:1.0:1761233348.813353:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233348.813359:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813383:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233348.813385:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813387:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233348.813392:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233348.813393:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233348.813395:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233348.813397:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233348.813399:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233348.813401:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233348.813406:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813407:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233348.813409:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813411:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233348.813416:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233348.813419:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233348.813423:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233348.813427:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233348.813431:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233348.813433:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813435:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233348.813437:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233348.813442:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813445:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233348.813450:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761233348.813454:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233348.813457:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233348.813460:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813463:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233348.813464:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233348.813465:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233348.813468:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233348.813471:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233348.813479:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245200768/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233348.813489:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233348.813491:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233348.813494:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245200768/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233348.813509:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233348.813511:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233348.813514:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5827) 00000800:00000200:3.1:1761233348.813594:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233348.813597:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761233348.814015:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233348.814031:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761233348.814262:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233348.814264:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1761233349.709142:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233349.709148:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233349.709153:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233351.757112:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233351.757116:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233351.757120:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233353.804991:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233353.804995:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233353.804998:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233353.933654:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233353.933824:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000400:00000200:0.0:1761233353.933950:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:0.0:1761233353.933969:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000800:00000200:3.1:1761233353.933977:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233353.934128:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000020:00000001:1.0:1761233355.853216:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233355.853220:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233355.853225:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1761233357.005264:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:0.0:1761233357.005332:0:88983:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1761233357.005346:0:88983:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:0.0:1761233357.005363:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233357.005369:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676778). 00000400:00000200:0.0:1761233357.005379:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233357.005387:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233357.005395:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:0.0:1761233357.005399:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:0.0:1761233357.005406:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233357.005416:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000345661c8 x1846784245200832/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233378 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233357.005467:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233357.005473:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000345661c8 x1846784245200832/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233378 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233357.005487:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233357.005490:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233357.005508:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233357.005512:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006315f304 (tot 1676530). 00000400:00000200:0.0:1761233357.005517:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233357.005523:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233357.005528:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cba99716 00000400:00000010:0.0:1761233357.005531:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cba99716. 00000100:00000001:0.0:1761233357.005536:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233357.005542:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000003346e953 x1846784245200896/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233378 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233357.005561:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233357.005567:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003346e953 x1846784245200896/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233378 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233357.005580:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233357.005617:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233357.005636:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233357.005644:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aaa22236 (tot 1676282). 00000400:00000200:0.0:1761233357.005653:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233357.005667:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233357.005676:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000f750763e 00000400:00000010:0.0:1761233357.005683:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000f750763e. 00000100:00000001:0.0:1761233357.005691:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233357.005704:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@0000000076f65824 x1846784245200960/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233378 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233357.005727:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233357.005738:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000076f65824 x1846784245200960/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233378 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233357.005779:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233357.005785:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000020:00000001:1.0:1761233357.901198:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233357.901201:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233357.901204:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233359.053055:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233359.053066:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233359.053068:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233359.053071:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233359.053074:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233359.053079:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233359.053086:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5828 00000100:00080000:0.0:1761233359.053090:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233359.053092:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053095:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233359.053097:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053101:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233359.053105:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233359.053107:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233359.053109:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233359.053112:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233359.053114:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233359.053116:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233359.053119:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053120:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233359.053122:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053123:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233359.053126:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233359.053129:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053132:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233359.053134:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233359.053137:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233359.053139:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053141:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233359.053143:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233359.053149:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233359.053152:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233359.053156:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233359.053157:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053160:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233359.053163:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233359.053167:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233359.053175:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233359.053182:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233359.053189:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245201024/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233359.053199:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233359.053219:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233359.053225:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245201024/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233359.053252:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053255:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233359.053257:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233359.053261:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233359.053261:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233359.053263:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233359.053281:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233359.053283:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233359.053285:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5828 00000100:00080000:0.0:1761233359.053286:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233359.053289:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053291:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233359.053294:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053296:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233359.053299:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233359.053300:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233359.053302:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233359.053305:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233359.053307:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233359.053310:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233359.053314:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053317:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233359.053320:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053323:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233359.053326:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233359.053330:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053333:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233359.053336:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233359.053344:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233359.053348:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053352:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233359.053358:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233359.053364:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053368:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233359.053375:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233359.053380:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233359.053401:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233359.053403:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053407:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233359.053411:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233359.053413:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233359.053419:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233359.053426:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233359.053435:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245201088/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233359.053446:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233359.053449:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233359.053455:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245201088/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233359.053468:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053471:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233359.053473:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233359.053479:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233359.053480:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233359.053482:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233359.053485:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233359.053489:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233359.053494:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5828 00000100:00080000:0.0:1761233359.053498:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233359.053505:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053510:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233359.053513:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053517:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233359.053521:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233359.053523:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233359.053526:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233359.053530:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233359.053535:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233359.053538:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233359.053542:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053545:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233359.053548:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053552:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233359.053555:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233359.053557:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233359.053559:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233359.053561:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233359.053563:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233359.053565:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053567:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233359.053569:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233359.053572:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053575:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233359.053581:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233359.053585:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233359.053589:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233359.053592:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053594:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233359.053596:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233359.053600:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233359.053606:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233359.053611:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233359.053620:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245201152/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233359.053641:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233359.053643:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233359.053650:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245201152/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233359.053662:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233359.053665:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233359.053670:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5838) 00000800:00000200:3.1:1761233359.053680:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233359.053684:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233359.054094:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233359.054096:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233359.054341:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233359.054343:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1761233359.949242:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233359.949248:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233359.949254:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233360.973017:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233360.973021:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233360.973025:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233361.997139:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233361.997142:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233361.997146:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233363.021029:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233363.021032:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233363.021035:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233364.045079:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233364.045082:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233364.045084:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1761233364.173206:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233364.173211:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233364.173214:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233364.173220:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173240:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233364.173244:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233364.173249:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233364.173254:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233364.173261:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b242e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245201216:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233364.173279:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233364.173284:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233364.173288:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233364.173293:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233364.173300:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d13d0f57. 02000000:00000001:1.0:1761233364.173305:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233364.173311:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233364.173320:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233364.173330:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201216, portal 25 00000100:00000001:1.0:1761233364.173336:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233364.173339:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091125952 : -104717618425664 : ffffa0c2878d50c0) 00000100:00000040:1.0:1761233364.173349:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233389 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173370:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233364.173376:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233364.173381:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233364.173386:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245201216, offset 0 00000400:00000200:1.0:1761233364.173394:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233364.173409:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233364.173415:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233364.173418:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233364.173421:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233364.173425:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11361:11361:256:4294967295] 192.168.201.124@tcp LPNI seq info [11361:11361:8:4294967295] 00000400:00000200:1.0:1761233364.173435:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233364.173441:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233364.173446:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233364.173453:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233364.173456:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761233364.173462:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233364.173465:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1761233364.173468:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233364.173470:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233364.173474:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233364.173478:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233364.173481:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233364.173483:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233364.173486:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233364.173489:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233389 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173495:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233364.173497:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233389 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173501:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173502:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233364.173506:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173507:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233364.173509:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173511:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233364.173513:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233364/real 1761233364] req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233364.173519:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233364.173520:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233364.173521:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233364.173522:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233364.173523:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233364.173524:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233364.173526:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233364.173530:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173534:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233364.173535:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173536:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233364.173538:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233364.173539:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173543:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233364.173546:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233364.173549:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233364.173551:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173552:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233364.173553:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173554:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233364.173555:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233364.173557:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233364.173558:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233364.173560:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233364.173562:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233364.173566:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b242e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245201216:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233364.173569:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173571:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233364.173572:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004b242e46 x1846784245201216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233364.173576:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233364.173578:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233364.173579:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d13d0f57. 02000000:00000001:1.0:1761233364.173581:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233364.173582:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233364.173583:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:1.0:1761233364.173584:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233364.173586:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000071d5cb10. 02000000:00000001:1.0:1761233364.173589:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233364.173591:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233364.173592:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004b242e46. 00000100:00000001:1.0:1761233364.173594:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233364.173595:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173596:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173602:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233364.173603:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173604:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233364.173605:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233364.173606:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233364.173608:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173612:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233364.173613:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233364.173614:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233364.173615:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233364.173616:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db341c88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201280:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233364.173618:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233364.173619:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233364.173620:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233364.173621:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233364.173623:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d13d0f57. 02000000:00000001:1.0:1761233364.173624:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233364.173626:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233364.173628:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233364.173629:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201280, portal 10 00000100:00000001:1.0:1761233364.173631:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233364.173631:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091132032 : -104717618419584 : ffffa0c2878d6880) 00000100:00000040:1.0:1761233364.173635:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233389 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173638:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233364.173640:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233364.173641:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233364.173642:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245201280, offset 0 00000400:00000200:1.0:1761233364.173645:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233364.173649:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233364.173651:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233364.173653:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233364.173654:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233364.173656:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11362:11362:256:4294967295] 192.168.201.124@tcp LPNI seq info [11362:11362:8:4294967295] 00000400:00000200:1.0:1761233364.173661:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233364.173664:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233364.173665:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233364.173668:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233364.173669:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761233364.173672:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233364.173673:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1761233364.173675:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233364.173676:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233364.173678:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233364.173680:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233364.173682:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233364.173683:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233364.173684:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233364.173685:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233389 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173689:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233364.173691:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233389 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173694:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173695:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233364.173696:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173697:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233364.173698:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173699:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233364.173700:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233364/real 1761233364] req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233364.173704:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233364.173706:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233364.173706:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233364.173707:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233364.173708:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233364.173710:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233364.173711:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233364.173715:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173718:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233364.173719:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173719:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233364.173720:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233364.173721:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173725:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233364.173726:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233364.173730:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233364.173731:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173732:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233364.173733:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173734:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233364.173735:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233364.173736:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233364.173738:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233364.173739:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233364.173740:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233364.173745:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db341c88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201280:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233364.173749:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233364.173750:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173754:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233364.173755:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233364.173756:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233364.173757:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233364.173758:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000266eaa93 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201344:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233364.173760:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233364.173760:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233364.173761:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233364.173762:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233364.173765:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233364.173766:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233364.173768:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233364.173769:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233364.173771:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201344, portal 4 00000100:00000001:1.0:1761233364.173772:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233364.173773:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091129600 : -104717618422016 : ffffa0c2878d5f00) 00000100:00000040:1.0:1761233364.173775:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233389 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173779:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233364.173780:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233364.173781:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233364.173782:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245201344, offset 0 00000400:00000200:1.0:1761233364.173785:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233364.173787:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233364.173789:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233364.173790:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233364.173791:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233364.173793:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11363:11363:256:4294967295] 192.168.201.124@tcp LPNI seq info [11363:11363:8:4294967295] 00000400:00000200:1.0:1761233364.173798:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233364.173801:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233364.173802:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233364.173805:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233364.173806:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1761233364.173808:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233364.173810:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1761233364.173812:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233364.173813:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233364.173815:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233364.173816:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233364.173818:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233364.173819:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233364.173820:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233364.173821:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233389 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173825:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233364.173826:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233389 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173829:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173830:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233364.173831:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173832:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233364.173833:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173834:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233364.173836:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233364/real 1761233364] req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233364.173840:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233364.173840:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233364.173841:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233364.173842:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233364.173869:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233364.173871:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233364.173873:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233364.173882:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173886:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233364.173887:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173888:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233364.173888:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233364.173890:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233364.173893:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233364.173895:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233364.173899:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233364.173900:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173901:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233364.173902:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173903:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233364.173903:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233364.173904:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233364.173906:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233364.173907:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233364.173908:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233364.173912:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000266eaa93 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201344:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233364.173914:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173915:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233364.173916:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000db341c88 x1846784245201280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233364.173921:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233364.173922:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233364.173923:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d13d0f57. 02000000:00000001:1.0:1761233364.173924:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233364.173925:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233364.173926:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:1.0:1761233364.173927:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233364.173928:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000df51b12a. 02000000:00000001:1.0:1761233364.173930:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233364.173930:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233364.173931:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000db341c88. 00000100:00000001:1.0:1761233364.173933:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233364.173934:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173934:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233364.173936:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000266eaa93 x1846784245201344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233389 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233364.173939:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233364.173940:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233364.173940:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233364.173942:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233364.173943:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233364.173943:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:1.0:1761233364.173944:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233364.173945:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000001c1b0e94. 02000000:00000001:1.0:1761233364.173946:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233364.173947:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233364.173947:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000266eaa93. 00000100:00000001:1.0:1761233364.173948:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233364.173949:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173950:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233364.173951:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233364.173952:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233364.173953:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233364.173954:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233365.069209:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233365.069214:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233365.069221:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233366.093202:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233366.093211:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233366.093218:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233367.117191:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233367.117197:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233367.117202:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233368.141113:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233368.141117:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233368.141121:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233369.165164:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233369.165169:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233369.165175:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1761233369.293352:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233369.293359:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233369.293364:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233369.293374:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.293393:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233369.293395:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233369.293402:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233369.293406:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233369.293415:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245201408:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233369.293426:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233369.293446:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233369.293449:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233369.293454:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233369.293463:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233369.293468:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233369.293479:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233369.293491:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233369.293504:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201408, portal 25 00000100:00000001:1.0:1761233369.293510:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233369.293514:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091112576 : -104717618439040 : ffffa0c2878d1c80) 00000100:00000040:1.0:1761233369.293524:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233394 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.293543:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233369.293552:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233369.293558:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233369.293564:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245201408, offset 0 00000400:00000200:1.0:1761233369.293576:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233369.293603:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233369.293614:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233369.293619:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233369.293622:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233369.293627:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11364:11364:256:4294967295] 192.168.201.124@tcp LPNI seq info [11364:11364:8:4294967295] 00000400:00000200:1.0:1761233369.293642:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233369.293649:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233369.293655:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233369.293663:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233369.293667:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761233369.293674:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233369.293678:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1761233369.293682:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233369.293685:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233369.293694:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233369.293699:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233369.293704:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233369.293707:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233369.293712:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233369.293715:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233394 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.293724:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233369.293726:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233394 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.293733:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.293734:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233369.293740:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.293742:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233369.293744:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.293748:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233369.293750:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233369/real 1761233369] req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233369.293759:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233369.293761:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233369.293762:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233369.293764:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233369.293765:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233369.293767:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233369.293769:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233369.293774:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.293779:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233369.293780:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.293782:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233369.293784:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233369.293787:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.293792:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233369.293796:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233369.293811:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233369.293813:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.293814:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233369.293815:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.293817:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233369.293819:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233369.293822:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233369.293825:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233369.293827:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233369.293830:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233369.293837:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245201408:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233369.293841:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.293879:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233369.293882:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a52d2576 x1846784245201408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233369.293888:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233369.293890:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233369.293893:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233369.293897:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233369.293899:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233369.293900:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:1.0:1761233369.293902:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233369.293904:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000009baf651a. 02000000:00000001:1.0:1761233369.293908:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233369.293909:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233369.293911:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a52d2576. 00000100:00000001:1.0:1761233369.293916:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233369.293917:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.293919:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.293929:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233369.293930:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.293932:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233369.293933:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233369.293935:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233369.293937:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.293942:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233369.293943:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233369.293945:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233369.293946:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233369.293948:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201472:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233369.293951:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233369.293953:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233369.293954:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233369.293955:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233369.293958:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233369.293959:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233369.293962:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233369.293964:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233369.293967:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201472, portal 10 00000100:00000001:1.0:1761233369.293968:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233369.293969:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091134464 : -104717618417152 : ffffa0c2878d7200) 00000100:00000040:1.0:1761233369.293974:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233394 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.293980:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233369.293982:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233369.293983:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233369.293985:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245201472, offset 0 00000400:00000200:1.0:1761233369.293988:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233369.293992:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233369.293995:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233369.293997:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233369.293999:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233369.294001:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11365:11365:256:4294967295] 192.168.201.124@tcp LPNI seq info [11365:11365:8:4294967295] 00000400:00000200:1.0:1761233369.294009:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233369.294013:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233369.294015:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233369.294018:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233369.294020:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761233369.294024:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233369.294026:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1761233369.294029:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233369.294030:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233369.294032:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233369.294035:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233369.294038:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233369.294039:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233369.294066:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233369.294069:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233394 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294075:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233369.294077:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233394 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294082:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.294108:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233369.294111:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.294113:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233369.294116:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.294118:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233369.294121:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233369/real 1761233369] req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233369.294131:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233369.294133:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233369.294135:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233369.294137:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233369.294138:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233369.294141:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233369.294144:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233369.294153:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294160:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233369.294162:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294164:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233369.294165:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233369.294168:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294177:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233369.294182:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233369.294187:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233369.294189:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294191:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233369.294192:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294193:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233369.294194:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233369.294196:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233369.294197:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233369.294199:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233369.294201:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233369.294206:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201472:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233369.294209:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233369.294211:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294216:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233369.294218:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233369.294220:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233369.294221:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233369.294223:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201536:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233369.294226:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233369.294227:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233369.294228:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233369.294230:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233369.294233:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000d13d0f57. 02000000:00000001:1.0:1761233369.294234:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233369.294236:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233369.294239:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233369.294242:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201536, portal 4 00000100:00000001:1.0:1761233369.294243:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233369.294244:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091118656 : -104717618432960 : ffffa0c2878d3440) 00000100:00000040:1.0:1761233369.294248:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233394 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294253:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233369.294255:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233369.294257:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233369.294258:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245201536, offset 0 00000400:00000200:1.0:1761233369.294262:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233369.294271:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233369.294273:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233369.294275:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233369.294277:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233369.294279:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11366:11366:256:4294967295] 192.168.201.124@tcp LPNI seq info [11366:11366:8:4294967295] 00000400:00000200:1.0:1761233369.294286:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233369.294289:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233369.294291:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233369.294295:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233369.294297:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1761233369.294300:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233369.294302:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1761233369.294305:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233369.294307:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233369.294309:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233369.294311:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233369.294313:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233369.294315:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233369.294316:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233369.294318:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233394 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294324:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233369.294326:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233394 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294331:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.294332:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233369.294334:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.294335:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233369.294336:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.294337:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233369.294339:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233369/real 1761233369] req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233369.294345:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233369.294346:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233369.294347:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233369.294348:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233369.294349:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233369.294350:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233369.294351:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233369.294356:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294360:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233369.294361:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294362:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233369.294363:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233369.294365:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233369.294369:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233369.294371:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233369.294375:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233369.294376:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294377:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233369.294379:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294380:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233369.294381:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233369.294383:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233369.294385:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233369.294387:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233369.294388:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233369.294393:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201536:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233369.294396:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294397:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233369.294398:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000194cd3a7 x1846784245201472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233369.294402:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233369.294403:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233369.294404:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233369.294406:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233369.294407:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233369.294408:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:1.0:1761233369.294410:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233369.294411:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074a27ae0. 02000000:00000001:1.0:1761233369.294413:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233369.294414:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233369.294415:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000194cd3a7. 00000100:00000001:1.0:1761233369.294417:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233369.294418:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294419:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233369.294421:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000009ecb9068 x1846784245201536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233394 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233369.294425:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233369.294427:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233369.294427:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000d13d0f57. 02000000:00000001:1.0:1761233369.294429:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233369.294441:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233369.294442:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:1.0:1761233369.294444:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233369.294445:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074003a94. 02000000:00000001:1.0:1761233369.294446:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233369.294447:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233369.294447:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000009ecb9068. 00000100:00000001:1.0:1761233369.294450:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233369.294451:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294452:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233369.294454:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233369.294455:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233369.294456:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233369.294457:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233370.189148:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233370.189153:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233370.189156:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233371.213103:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233371.213107:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233371.213111:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233372.237206:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233372.237214:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233372.237222:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233373.261162:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233373.261167:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233373.261170:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233374.285096:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233374.285099:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233374.285102:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1761233374.413274:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233374.413280:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233374.413286:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233374.413299:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.413359:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233374.413362:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233374.413369:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233374.413374:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233374.413391:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245201600:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233374.413401:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233374.413409:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233374.413413:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233374.413423:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233374.413436:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233374.413441:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233374.413451:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233374.413470:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233374.413482:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201600, portal 25 00000100:00000001:1.0:1761233374.413490:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233374.413493:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091118656 : -104717618432960 : ffffa0c2878d3440) 00000100:00000040:1.0:1761233374.413498:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233399 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.413510:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233374.413516:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233374.413522:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233374.413530:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245201600, offset 0 00000400:00000200:1.0:1761233374.413543:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233374.413572:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233374.413583:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233374.413590:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233374.413597:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233374.413609:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11367:11367:256:4294967295] 192.168.201.124@tcp LPNI seq info [11367:11367:8:4294967295] 00000400:00000200:1.0:1761233374.413651:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233374.413663:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233374.413671:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233374.413693:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233374.413700:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761233374.413715:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233374.413723:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1761233374.413729:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233374.413731:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233374.413735:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233374.413739:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233374.413742:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233374.413743:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233374.413748:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233374.413754:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233399 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.413763:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233374.413765:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233399 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.413769:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.413770:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233374.413774:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.413775:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233374.413777:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.413779:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233374.413781:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233374/real 1761233374] req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233374.413786:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233374.413787:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233374.413788:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233374.413789:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233374.413790:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233374.413791:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233374.413793:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233374.413797:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.413801:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233374.413802:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.413803:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233374.413805:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233374.413807:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.413811:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233374.413814:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233374.413818:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233374.413819:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.413820:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233374.413821:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.413823:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233374.413824:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233374.413826:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233374.413828:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233374.413830:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233374.413831:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233374.413836:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245201600:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233374.413839:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.413840:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233374.413879:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000009ecb9068 x1846784245201600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233374.413884:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233374.413886:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233374.413888:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233374.413891:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233374.413892:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233374.413894:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:1.0:1761233374.413896:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233374.413897:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062959873. 02000000:00000001:1.0:1761233374.413901:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233374.413903:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233374.413904:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000009ecb9068. 00000100:00000001:1.0:1761233374.413907:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233374.413908:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.413910:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.413927:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233374.413929:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.413930:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233374.413932:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761233374.413933:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233374.413935:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.413939:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233374.413940:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233374.413941:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233374.413943:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233374.413944:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201664:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233374.413946:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233374.413947:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233374.413948:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233374.413949:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233374.413951:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233374.413952:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233374.413954:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233374.413956:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233374.413957:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201664, portal 10 00000100:00000001:1.0:1761233374.413959:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233374.413973:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091134464 : -104717618417152 : ffffa0c2878d7200) 00000100:00000040:1.0:1761233374.413977:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233399 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.413981:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233374.413983:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233374.413984:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233374.413985:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245201664, offset 0 00000400:00000200:1.0:1761233374.413988:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233374.413992:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233374.413994:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233374.413996:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233374.413997:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233374.413999:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11368:11368:256:4294967295] 192.168.201.124@tcp LPNI seq info [11368:11368:8:4294967295] 00000400:00000200:1.0:1761233374.414004:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233374.414007:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233374.414009:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233374.414011:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233374.414013:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761233374.414015:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233374.414017:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1761233374.414019:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233374.414020:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233374.414022:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233374.414024:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233374.414025:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233374.414026:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233374.414027:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233374.414029:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233399 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414033:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233374.414035:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233399 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414038:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.414039:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233374.414041:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.414042:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233374.414043:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.414044:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233374.414045:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233374/real 1761233374] req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233374.414049:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233374.414050:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233374.414051:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233374.414052:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233374.414053:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233374.414054:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233374.414055:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233374.414059:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414062:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233374.414063:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414064:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233374.414065:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233374.414066:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414070:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233374.414071:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233374.414074:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233374.414076:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414076:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233374.414077:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414078:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233374.414079:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233374.414080:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233374.414082:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233374.414084:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233374.414085:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233374.414089:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201664:192.168.201.124@tcp:38: 00000100:00000001:1.0:1761233374.414093:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761233374.414095:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414099:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233374.414100:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233374.414101:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233374.414102:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233374.414103:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201728:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233374.414105:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761233374.414106:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761233374.414106:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233374.414107:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761233374.414109:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f43b35ab. 02000000:00000001:1.0:1761233374.414110:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1761233374.414112:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000400:00000010:1.0:1761233374.414113:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000200:1.0:1761233374.414114:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201728, portal 4 00000100:00000001:1.0:1761233374.414116:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761233374.414116:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091112576 : -104717618439040 : ffffa0c2878d1c80) 00000100:00000040:1.0:1761233374.414119:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233399 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414122:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761233374.414124:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:1.0:1761233374.414125:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233374.414126:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245201728, offset 0 00000400:00000200:1.0:1761233374.414128:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:1.0:1761233374.414130:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:1.0:1761233374.414132:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:1.0:1761233374.414133:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233374.414135:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233374.414136:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11369:11369:256:4294967295] 192.168.201.124@tcp LPNI seq info [11369:11369:8:4294967295] 00000400:00000200:1.0:1761233374.414141:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:1.0:1761233374.414144:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233374.414146:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233374.414148:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.0:1761233374.414149:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1761233374.414151:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233374.414152:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1761233374.414155:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233374.414155:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000400:00000200:1.0:1761233374.414157:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1761233374.414158:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1761233374.414160:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233374.414160:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233374.414162:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233374.414163:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233399 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414167:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233374.414168:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233399 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414171:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.414172:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233374.414173:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.414174:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233374.414175:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.414176:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233374.414177:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233374/real 1761233374] req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233374.414182:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233374.414182:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233374.414183:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233374.414184:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233374.414185:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233374.414186:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233374.414187:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233374.414195:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414198:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233374.414199:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414200:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233374.414201:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233374.414202:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233374.414205:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233374.414207:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233374.414210:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233374.414211:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414212:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233374.414212:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414213:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233374.414214:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233374.414215:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233374.414216:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233374.414217:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233374.414219:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233374.414222:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245201728:192.168.201.124@tcp:8: 00000100:00000001:1.0:1761233374.414224:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414225:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233374.414226:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000194cd3a7 x1846784245201664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233374.414230:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233374.414231:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233374.414232:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f654abd8. 02000000:00000001:1.0:1761233374.414234:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233374.414235:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233374.414235:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:1.0:1761233374.414236:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233374.414237:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000296782d7. 02000000:00000001:1.0:1761233374.414239:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233374.414240:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233374.414241:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000194cd3a7. 00000100:00000001:1.0:1761233374.414242:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233374.414243:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414244:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233374.414245:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a52d2576 x1846784245201728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233399 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233374.414248:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233374.414249:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233374.414250:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f43b35ab. 02000000:00000001:1.0:1761233374.414251:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233374.414252:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233374.414252:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:1.0:1761233374.414253:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233374.414253:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000be06bfbf. 02000000:00000001:1.0:1761233374.414255:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233374.414255:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233374.414256:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a52d2576. 00000100:00000001:1.0:1761233374.414258:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233374.414259:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414260:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233374.414262:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233374.414262:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233374.414263:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233374.414264:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233375.309101:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233375.309120:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233375.309124:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233376.333016:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233376.333020:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233376.333023:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233377.357114:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233377.357118:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233377.357121:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233378.381013:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233378.381017:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233378.381022:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233379.405052:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233379.405054:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233379.405056:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233379.533038:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233379.533055:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233379.533057:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233379.533061:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233379.533064:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233379.533072:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233379.533082:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5848 00000100:00080000:0.0:1761233379.533086:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233379.533090:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533093:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233379.533096:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533099:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233379.533103:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233379.533105:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233379.533109:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233379.533112:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233379.533116:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233379.533130:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233379.533134:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533136:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233379.533138:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533140:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233379.533144:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233379.533149:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533152:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233379.533155:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233379.533160:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233379.533163:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533166:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233379.533168:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233379.533182:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002ca01b78. 00000020:00000040:0.0:1761233379.533187:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233379.533192:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233379.533193:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533197:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233379.533217:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233379.533221:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233379.533234:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233379.533242:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233379.533250:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233379.533262:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233379.533265:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233379.533270:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002ca01b78] to pc [ptlrpcd_rcv+-1] req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233379.533295:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533297:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233379.533300:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233379.533303:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233379.533305:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233379.533306:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233379.533307:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233379.533310:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233379.533312:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5848 00000100:00080000:0.0:1761233379.533315:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233379.533318:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533320:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233379.533323:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533325:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233379.533327:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233379.533328:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233379.533330:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233379.533331:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233379.533333:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233379.533335:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233379.533338:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533340:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233379.533341:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533343:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233379.533345:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233379.533347:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533350:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233379.533351:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233379.533358:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233379.533360:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533363:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233379.533367:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233379.533371:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533373:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233379.533378:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000095969832. 00000020:00000040:0.0:1761233379.533381:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233379.533384:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233379.533386:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533388:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233379.533390:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233379.533392:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233379.533397:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233379.533402:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233379.533408:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000095969832 x1846784245201856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233379.533409:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233379.533416:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233379.533418:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233379.533420:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1761233379.533422:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233379.533431:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000095969832] to pc [ptlrpcd_rcv+-1] req@0000000095969832 x1846784245201856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:1.0:1761233379.533432:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233379.533481:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761233379.533483:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533487:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233379.533488:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761233379.533491:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533492:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233379.533496:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1761233379.533497:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ca01b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245201792:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233379.533508:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1761233379.533510:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233379.533513:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1761233379.533514:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1761233379.533517:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1761233379.533517:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233379.533520:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1761233379.533521:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233379.533528:0:64357:0:(import.c:519:import_select_connection()) Process entered 02000000:00000010:1.0:1761233379.533530:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f43b35ab. 00000100:00080000:0.0:1761233379.533534:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5848 02000000:00000001:1.0:1761233379.533535:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1761233379.533537:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1761233379.533544:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000028255bec. 00000100:00000040:0.0:1761233379.533545:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533554:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233379.533560:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000400:00000010:1.0:1761233379.533560:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a6fd6a6b. 00000100:00000001:0.0:1761233379.533573:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000200:1.0:1761233379.533576:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245201792, portal 25 00000020:00000001:0.0:1761233379.533578:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000100:00000001:1.0:1761233379.533583:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000020:00000040:0.0:1761233379.533585:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000100:00000001:1.0:1761233379.533590:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246321280 : -104717463230336 : ffffa0c290cd6880) 00000020:00000001:0.0:1761233379.533591:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233379.533596:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233379.533600:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233379.533602:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233404 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000020:00000001:0.0:1761233379.533604:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233379.533615:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233379.533621:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233379.533625:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233379.533627:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:0.0:1761233379.533629:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233379.533632:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00000040:1.0:1761233379.533638:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00080000:0.0:1761233379.533642:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000010:1.0:1761233379.533643:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000fe6afb1a. 00000100:00000200:1.0:1761233379.533647:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245201792, offset 0 00000100:00000001:0.0:1761233379.533649:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233379.533654:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1761233379.533654:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:0.0:1761233379.533661:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233379.533667:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533670:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000400:00000200:1.0:1761233379.533672:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000008:00100000:0.0:1761233379.533674:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1761233379.533680:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000008:00000001:0.0:1761233379.533681:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533687:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1761233379.533688:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1761233379.533693:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:1.0:1761233379.533697:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11370:11370:256:4294967295] 192.168.201.124@tcp LPNI seq info [11370:11370:8:4294967295] 00000100:00000010:0.0:1761233379.533698:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233379.533707:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233379.533715:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233379.533718:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1761233379.533718:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:0.0:1761233379.533725:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233379.533728:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.0:1761233379.533728:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 02000000:00000001:0.0:1761233379.533736:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1761233379.533736:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dcd694ea (tot 1676530). 00000800:00000200:1.0:1761233379.533749:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000010:0.0:1761233379.533750:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000800:00000200:1.0:1761233379.533757:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000100:00000001:0.0:1761233379.533759:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1761233379.533765:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:1.1:1761233379.533768:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000100:00100000:0.0:1761233379.533772:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245201920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000100:1.0:1761233379.533779:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:1.0:1761233379.533781:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dcd694ea (tot 1676282). 00000100:00000001:0.0:1761233379.533804:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1761233379.533805:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1761233379.533808:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1761233379.533812:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1761233379.533816:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245201920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:1.0:1761233379.533817:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000fe6afb1a 00000400:00000010:1.0:1761233379.533822:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000fe6afb1a. 00000100:00000001:1.0:1761233379.533830:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233379.533837:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233404 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233379.533850:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233379.533855:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233404 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233379.533912:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233379.533916:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233379.533921:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5858) 00000100:00000001:1.0:1761233379.533936:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233379.533938:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761233379.533946:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233379.533948:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761233379.533952:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233379.533955:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1761233379.533960:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233379/real 1761233379] req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233404 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:1.0:1761233379.533970:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1761233379.533971:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1761233379.533973:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000028255bec. 00000400:00000200:1.0:1761233379.533974:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a6fd6a6b 00000400:00000010:1.0:1761233379.533975:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a6fd6a6b. 00000100:00000001:1.0:1761233379.533977:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:1.0:1761233379.533979:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233404 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:1.0:1761233379.533984:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233404 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233379.533988:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761233379.533988:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233379.533990:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233379.533991:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1761233379.533993:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233404 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233379.533997:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761233379.534000:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233404 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233379.534004:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1761233379.534005:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233379.534006:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1761233379.534007:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233379.534008:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1761233379.534010:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1761233379.534012:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1761233379.534014:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:1.0:1761233379.534016:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1761233379.534018:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233404 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:1.0:1761233379.534023:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ca01b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245201792:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761233379.534026:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233379.534028:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233379.534029:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002ca01b78 x1846784245201792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233404 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:1.0:1761233379.534033:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1761233379.534034:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1761233379.534036:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f43b35ab. 02000000:00000001:1.0:1761233379.534038:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1761233379.534039:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:1.0:1761233379.534041:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:1.0:1761233379.534042:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:1.0:1761233379.534043:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:1.0:1761233379.534046:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1761233379.534047:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1761233379.534048:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002ca01b78. 00000100:00000001:1.0:1761233379.534050:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1761233379.534051:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233379.534053:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761233379.534059:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1761233379.534060:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233379.534062:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761233379.534063:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000800:00000200:2.1:1761233379.534251:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233379.534260:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761233379.534516:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233379.534518:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1761233380.429039:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233380.429043:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233380.429046:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233381.453080:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233381.453083:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233381.453086:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233382.477038:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233382.477042:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233382.477045:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233383.501055:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233383.501059:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233383.501062:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233384.525035:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233384.525038:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233384.525041:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233384.653582:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233384.653591:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1761233384.653951:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233384.653953:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1761233384.654108:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233384.654110:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000020:00000001:1.0:1761233385.549073:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233385.549077:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233385.549080:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233386.573092:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233386.573097:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233386.573102:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233387.597093:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233387.597100:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233387.597106:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233388.621151:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233388.621155:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233388.621158:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233389.645136:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233389.645140:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233389.645143:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233389.773503:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233389.773506:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1761233389.773960:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233389.773962:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1761233389.774200:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233389.774203:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000020:00000001:1.0:1761233390.669017:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233390.669020:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233390.669022:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233391.693069:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233391.693073:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233391.693076:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233392.717104:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233392.717109:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233392.717113:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233393.741056:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233393.741061:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233393.741065:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233394.765085:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233394.765089:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233394.765093:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233394.893612:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233394.893636:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233394.894117:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233394.894119:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233394.894447:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233394.894449:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1761233395.789075:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233395.789079:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233395.789083:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233396.813106:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233396.813111:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233396.813115:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233397.837169:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233397.837173:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233397.837177:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233398.861178:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233398.861188:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233398.861198:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233399.885104:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233399.885108:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233399.885111:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233400.014116:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233400.014121:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761233400.014521:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233400.014523:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761233400.014838:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233400.014841:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233400.909120:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233400.909123:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233400.909126:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233401.933092:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233401.933096:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233401.933100:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233402.957111:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233402.957113:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233402.957116:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233403.981102:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233403.981106:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233403.981110:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233405.005084:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233405.005089:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233405.005093:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233405.133045:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233405.133055:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233405.133056:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233405.133059:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233405.133061:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233405.133065:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233405.133072:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5874 00000100:00080000:0.0:1761233405.133074:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233405.133076:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133078:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233405.133080:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133082:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233405.133084:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233405.133085:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233405.133087:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233405.133090:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233405.133091:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233405.133093:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233405.133095:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133096:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233405.133097:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133098:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233405.133100:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233405.133102:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133104:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233405.133106:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233405.133108:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233405.133109:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133111:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233405.133112:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233405.133120:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233405.133122:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233405.133126:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233405.133126:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133129:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233405.133131:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233405.133134:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233405.133142:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233405.133147:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233405.133152:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245202752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233405.133159:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233405.133160:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233405.133163:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245202752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233405.133201:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133203:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233405.133204:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233405.133207:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233405.133207:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233405.133208:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233405.133209:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233405.133211:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233405.133212:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5874 00000100:00080000:0.0:1761233405.133213:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233405.133215:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133216:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233405.133217:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133222:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233405.133224:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233405.133224:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233405.133225:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233405.133226:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233405.133228:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233405.133230:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233405.133232:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133233:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233405.133234:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133236:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233405.133238:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233405.133241:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133244:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233405.133246:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233405.133252:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233405.133254:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133256:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233405.133260:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233405.133267:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133270:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233405.133276:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233405.133281:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233405.133283:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233405.133285:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133295:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233405.133297:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233405.133300:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233405.133305:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233405.133312:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233405.133325:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245202816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233405.133340:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233405.133343:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233405.133352:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245202816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233405.133366:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133371:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233405.133374:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233405.133380:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233405.133383:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233405.133387:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233405.133389:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233405.133392:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233405.133396:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5874 00000100:00080000:0.0:1761233405.133403:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233405.133408:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133414:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233405.133417:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133420:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233405.133425:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233405.133428:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233405.133433:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233405.133437:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233405.133441:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233405.133448:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233405.133459:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133466:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233405.133471:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133496:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233405.133499:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233405.133503:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233405.133505:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233405.133507:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233405.133512:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233405.133520:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133524:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233405.133528:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233405.133536:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133539:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233405.133544:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233405.133551:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233405.133555:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233405.133557:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133558:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233405.133560:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233405.133562:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233405.133566:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233405.133571:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233405.133580:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245202880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233405.133593:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233405.133596:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233405.133604:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245202880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233405.133629:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233405.133634:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233405.133639:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5884) 00000800:00000200:2.1:1761233405.133676:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233405.133678:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761233405.133956:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233405.133958:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761233405.134099:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233405.134100:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1761233406.029128:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233406.029131:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233406.029135:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233407.053150:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233407.053156:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233407.053162:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233408.077064:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233408.077068:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233408.077071:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233409.101064:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233409.101069:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233409.101083:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233410.125038:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233410.125042:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233410.125045:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233410.253069:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233410.253081:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233410.253083:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233410.253086:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233410.253088:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233410.253094:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233410.253101:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5879 00000100:00080000:1.0:1761233410.253104:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233410.253106:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253109:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233410.253111:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253113:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233410.253116:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233410.253118:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233410.253120:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233410.253123:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233410.253125:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233410.253127:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233410.253129:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253130:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233410.253132:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253133:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233410.253134:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233410.253137:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253139:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233410.253141:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233410.253146:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233410.253147:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253149:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233410.253151:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233410.253164:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761233410.253167:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233410.253172:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233410.253173:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253176:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233410.253179:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233410.253183:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233410.253192:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233410.253197:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233410.253204:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245202944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233410.253213:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233410.253215:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233410.253218:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245202944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233410.253237:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253239:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233410.253240:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233410.253243:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233410.253244:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233410.253245:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233410.253246:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233410.253247:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233410.253249:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5879 00000100:00080000:1.0:1761233410.253250:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233410.253252:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253253:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233410.253254:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253254:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233410.253257:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233410.253258:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233410.253259:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233410.253260:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233410.253262:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233410.253263:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233410.253265:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253266:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233410.253267:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253268:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233410.253269:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233410.253271:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253274:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233410.253275:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233410.253282:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233410.253284:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253286:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233410.253290:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233410.253295:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253298:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233410.253303:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761233410.253306:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233410.253311:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233410.253312:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253316:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233410.253317:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233410.253319:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233410.253325:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233410.253332:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233410.253339:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245203008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233410.253350:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233410.253354:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233410.253362:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245203008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233410.253372:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253376:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233410.253379:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233410.253383:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233410.253384:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233410.253387:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233410.253390:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233410.253393:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233410.253409:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5879 00000100:00080000:1.0:1761233410.253411:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233410.253414:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253417:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233410.253419:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253420:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233410.253421:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233410.253422:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233410.253424:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233410.253426:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233410.253429:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233410.253430:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233410.253432:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253433:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233410.253435:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253438:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233410.253441:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233410.253444:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233410.253447:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233410.253450:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233410.253453:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233410.253456:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253458:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233410.253460:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233410.253463:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253466:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233410.253470:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000020:00000040:1.0:1761233410.253473:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233410.253476:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233410.253477:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253480:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233410.253481:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233410.253483:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233410.253487:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233410.253491:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233410.253496:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245203072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233410.253502:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233410.253503:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233410.253507:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245203072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233410.253513:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233410.253515:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233410.253518:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5889) 00000800:00000200:2.1:1761233410.253601:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233410.253604:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761233410.253884:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233410.253886:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761233410.254041:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233410.254042:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1761233411.149083:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233411.149089:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233411.149095:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233412.173072:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233412.173075:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233412.173078:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233413.197052:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233413.197055:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233413.197057:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233415.245071:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233415.245075:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233415.245079:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233415.372967:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233415.372976:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233415.372977:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233415.372979:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233415.372981:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233415.372985:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233415.372990:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5884 00000100:00080000:1.0:1761233415.372993:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233415.372995:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.372998:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233415.372999:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373000:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233415.373003:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233415.373004:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233415.373006:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233415.373008:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233415.373010:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233415.373011:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233415.373013:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373014:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233415.373016:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373017:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233415.373019:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233415.373025:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373027:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233415.373029:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233415.373032:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233415.373033:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373035:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233415.373038:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233415.373046:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761233415.373048:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233415.373051:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233415.373052:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373055:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233415.373057:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233415.373059:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233415.373065:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233415.373071:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233415.373077:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245203136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233415.373086:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233415.373088:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233415.373093:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245203136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233415.373118:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373120:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233415.373122:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233415.373126:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233415.373127:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233415.373128:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233415.373129:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233415.373131:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233415.373134:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5884 00000100:00080000:1.0:1761233415.373136:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233415.373138:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373140:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233415.373142:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373144:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233415.373146:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233415.373147:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233415.373149:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233415.373151:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233415.373153:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233415.373155:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233415.373159:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373176:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233415.373179:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373181:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233415.373183:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233415.373186:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373190:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233415.373193:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233415.373201:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233415.373206:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373210:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233415.373214:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233415.373219:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373223:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233415.373229:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761233415.373233:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233415.373239:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233415.373241:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373244:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233415.373246:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233415.373249:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233415.373256:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233415.373262:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233415.373271:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245203200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233415.373284:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233415.373287:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233415.373293:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245203200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233415.373306:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373310:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233415.373314:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233415.373334:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233415.373336:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233415.373339:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233415.373341:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233415.373345:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233415.373350:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5884 00000100:00080000:1.0:1761233415.373354:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233415.373361:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373365:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233415.373368:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373372:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233415.373377:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233415.373378:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233415.373382:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233415.373384:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233415.373389:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233415.373391:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233415.373396:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373398:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233415.373403:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373407:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233415.373410:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233415.373416:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233415.373420:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233415.373423:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233415.373428:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233415.373434:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373438:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233415.373441:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233415.373445:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373449:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233415.373454:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761233415.373459:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233415.373463:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233415.373465:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373469:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233415.373470:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233415.373473:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233415.373480:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233415.373486:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233415.373494:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245203264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:2.1:1761233415.373503:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:1.0:1761233415.373525:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000800:00000200:2.1:1761233415.373526:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1761233415.373528:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233415.373535:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245203264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233415.373547:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233415.373550:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233415.373556:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5894) 00000800:00000200:2.1:1761233415.373923:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233415.373925:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761233415.374169:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233415.374171:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1761233417.293081:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233417.293085:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233417.293088:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233419.341010:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233419.341013:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233419.341015:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233420.492973:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233420.492982:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233420.492984:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233420.492986:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233420.492988:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233420.492992:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233420.492997:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5889 00000100:00080000:0.0:1761233420.492999:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233420.493001:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493004:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233420.493005:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493007:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233420.493009:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233420.493010:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233420.493012:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233420.493015:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233420.493017:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233420.493018:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233420.493020:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493021:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233420.493022:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493023:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233420.493025:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233420.493027:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493029:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233420.493030:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233420.493032:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233420.493034:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493035:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233420.493037:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233420.493043:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233420.493045:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233420.493048:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233420.493049:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493051:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233420.493052:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233420.493055:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233420.493061:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233420.493065:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233420.493071:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245203328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233420.493078:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233420.493079:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233420.493082:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245203328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233420.493102:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493104:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233420.493106:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233420.493108:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233420.493109:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233420.493110:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233420.493111:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233420.493112:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233420.493113:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5889 00000100:00080000:0.0:1761233420.493115:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233420.493116:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493117:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233420.493118:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493119:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233420.493120:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233420.493121:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233420.493122:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233420.493123:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233420.493124:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233420.493125:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233420.493127:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493127:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233420.493128:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493129:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233420.493130:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233420.493132:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493133:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233420.493134:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233420.493138:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233420.493139:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493140:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233420.493142:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233420.493144:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493145:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233420.493148:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233420.493149:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233420.493151:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233420.493151:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493152:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233420.493153:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233420.493154:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233420.493158:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233420.493160:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233420.493162:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245203392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233420.493167:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233420.493168:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233420.493170:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245203392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233420.493174:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493175:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233420.493176:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233420.493178:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233420.493180:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233420.493181:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233420.493183:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233420.493186:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233420.493190:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5889 00000100:00080000:0.0:1761233420.493191:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233420.493195:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493199:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233420.493201:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493204:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233420.493208:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233420.493209:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233420.493210:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233420.493211:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233420.493213:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233420.493216:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233420.493221:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493222:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233420.493225:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493227:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233420.493231:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233420.493247:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233420.493252:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233420.493254:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233420.493257:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233420.493260:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493262:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233420.493263:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233420.493265:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493268:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233420.493270:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233420.493273:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233420.493274:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233420.493275:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493276:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233420.493278:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233420.493280:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233420.493286:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233420.493289:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233420.493294:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245203456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233420.493299:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233420.493301:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233420.493304:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245203456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233420.493313:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233420.493316:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233420.493320:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5899) 00000800:00000200:2.1:1761233420.493390:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233420.493484:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233420.493497:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000400:00000200:0.0:1761233420.493559:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:0.0:1761233420.493593:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000800:00000200:2.1:1761233420.493629:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000020:00000001:1.0:1761233421.389072:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233421.389076:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233421.389081:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233423.437056:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233423.437062:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233423.437069:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1761233423.565261:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:0.0:1761233423.565324:0:88982:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1761233423.565337:0:88982:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:0.0:1761233423.565354:0:88982:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233423.565360:0:88982:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676778). 00000400:00000200:0.0:1761233423.565369:0:88982:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233423.565375:0:88982:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233423.565385:0:88982:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000053c991f 00000400:00000010:0.0:1761233423.565388:0:88982:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000053c991f. 00000100:00000001:0.0:1761233423.565397:0:88982:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233423.565406:0:88982:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000e728cb50 x1846784245203328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233445 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233423.565441:0:88982:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233423.565446:0:88982:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000e728cb50 x1846784245203328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233445 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233423.565456:0:88982:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233423.565459:0:88982:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233423.565478:0:88982:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233423.565482:0:88982:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007ceb0618 (tot 1676530). 00000400:00000200:0.0:1761233423.565487:0:88982:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233423.565495:0:88982:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233423.565501:0:88982:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a7ac5cb3 00000400:00000010:0.0:1761233423.565506:0:88982:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a7ac5cb3. 00000100:00000001:0.0:1761233423.565510:0:88982:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233423.565515:0:88982:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000dc434e0c x1846784245203392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233445 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233423.565529:0:88982:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233423.565538:0:88982:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000dc434e0c x1846784245203392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233445 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233423.565551:0:88982:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233423.565557:0:88982:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233423.565569:0:88982:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233423.565574:0:88982:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:0.0:1761233423.565584:0:88982:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233423.565594:0:88982:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233423.565613:0:88982:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000035e33961 00000400:00000010:0.0:1761233423.565620:0:88982:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000035e33961. 00000100:00000001:0.0:1761233423.565626:0:88982:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233423.565637:0:88982:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000039bf264 x1846784245203456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233445 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233423.565686:0:88982:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233423.565698:0:88982:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000039bf264 x1846784245203456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233445 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233423.565719:0:88982:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233423.565724:0:88982:0:(events.c:93:request_out_callback()) Process leaving 00000020:00000001:1.0:1761233425.485012:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233425.485016:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233425.485019:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233425.613071:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233425.613081:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233425.613083:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233425.613085:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233425.613087:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233425.613096:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233425.613102:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5894 00000100:00080000:0.0:1761233425.613104:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233425.613106:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613110:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233425.613111:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613113:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233425.613116:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233425.613118:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233425.613120:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233425.613123:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233425.613125:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233425.613127:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233425.613130:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613132:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233425.613134:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613136:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233425.613138:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233425.613140:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613143:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233425.613146:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233425.613148:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233425.613151:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613153:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233425.613170:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233425.613177:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233425.613180:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233425.613184:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233425.613186:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613189:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233425.613191:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233425.613195:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233425.613202:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233425.613207:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233425.613214:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245203520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233425.613227:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233425.613228:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233425.613231:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245203520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233425.613245:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613247:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233425.613248:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233425.613251:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233425.613251:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233425.613252:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233425.613253:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233425.613255:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233425.613257:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5894 00000100:00080000:0.0:1761233425.613258:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233425.613260:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613261:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233425.613262:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613263:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233425.613264:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233425.613265:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233425.613266:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233425.613267:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233425.613268:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233425.613270:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233425.613271:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613273:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233425.613274:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613276:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233425.613277:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233425.613280:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613283:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233425.613284:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233425.613289:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233425.613291:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613292:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233425.613296:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233425.613300:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613303:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233425.613307:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233425.613310:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233425.613314:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233425.613315:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613319:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233425.613320:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233425.613321:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233425.613327:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233425.613330:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233425.613336:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245203584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233425.613344:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233425.613346:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233425.613352:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245203584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233425.613362:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613365:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233425.613369:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233425.613372:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233425.613373:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233425.613375:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233425.613377:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233425.613381:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233425.613384:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5894 00000100:00080000:0.0:1761233425.613386:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233425.613391:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613393:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233425.613395:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613401:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233425.613404:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233425.613407:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233425.613411:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233425.613415:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233425.613420:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233425.613425:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233425.613435:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613440:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233425.613445:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613449:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233425.613470:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233425.613476:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233425.613479:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233425.613482:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233425.613485:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233425.613489:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613493:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233425.613496:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233425.613501:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613505:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233425.613511:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761233425.613517:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233425.613523:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233425.613525:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613529:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233425.613531:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233425.613534:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233425.613541:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233425.613547:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233425.613555:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245203648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233425.613565:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233425.613568:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233425.613575:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245203648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233425.613589:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233425.613593:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:2.1:1761233425.613595:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000040:0.0:1761233425.613599:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5904) 00000800:00000200:2.1:1761233425.613600:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233425.614008:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233425.614010:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233425.614277:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233425.614279:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1761233427.533178:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233427.533185:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233427.533190:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233429.581087:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233429.581091:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233429.581125:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233430.732923:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233430.732932:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233430.732934:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233430.732936:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233430.732938:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233430.732942:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233430.732947:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5899 00000100:00080000:0.0:1761233430.732949:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233430.732951:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.732953:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233430.732955:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.732957:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233430.732959:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233430.732960:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233430.732962:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233430.732965:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233430.732966:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233430.732967:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233430.732969:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.732971:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233430.732972:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.732974:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233430.732975:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233430.732978:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233430.732979:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233430.732981:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233430.732983:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233430.732985:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.732987:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233430.732988:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233430.732996:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233430.732998:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233430.733001:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233430.733002:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733004:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233430.733007:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233430.733009:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233430.733014:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233430.733020:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233430.733028:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245203712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233430.733037:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233430.733040:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233430.733044:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245203712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233430.733064:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733066:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233430.733068:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233430.733072:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233430.733072:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233430.733093:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233430.733095:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233430.733099:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233430.733102:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5899 00000100:00080000:0.0:1761233430.733105:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233430.733108:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.733110:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233430.733113:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.733116:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233430.733119:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233430.733122:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233430.733124:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233430.733126:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233430.733130:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233430.733140:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233430.733148:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.733152:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233430.733157:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.733161:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233430.733165:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233430.733170:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233430.733174:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233430.733178:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233430.733185:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233430.733189:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733192:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233430.733196:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233430.733201:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733205:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233430.733212:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233430.733217:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233430.733223:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233430.733226:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733231:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233430.733233:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233430.733236:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233430.733244:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233430.733250:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233430.733257:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245203776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233430.733269:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233430.733271:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233430.733278:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245203776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233430.733300:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733304:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233430.733306:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233430.733314:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233430.733316:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233430.733319:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233430.733321:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233430.733326:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233430.733333:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5899 00000100:00080000:0.0:1761233430.733338:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233430.733345:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000800:00000200:2.1:1761233430.733347:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:0.0:1761233430.733348:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1761233430.733351:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000100:00000040:0.0:1761233430.733354:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.733359:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233430.733363:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233430.733366:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233430.733368:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233430.733371:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233430.733377:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233430.733381:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233430.733388:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.733392:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233430.733397:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233430.733400:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233430.733405:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233430.733409:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233430.733415:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233430.733417:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233430.733420:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233430.733424:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733427:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233430.733429:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233430.733435:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733438:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233430.733442:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761233430.733448:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233430.733453:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233430.733455:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733459:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233430.733461:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233430.733463:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233430.733469:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233430.733483:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233430.733492:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245203840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233430.733507:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233430.733509:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233430.733521:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245203840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233430.733543:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233430.733548:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233430.733554:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5909) 00000800:00000200:2.1:1761233430.733674:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233430.733675:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761233430.733835:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233430.733837:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1761233431.629091:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233431.629095:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233431.629097:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233433.677144:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233433.677149:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233433.677153:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233435.725119:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233435.725123:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233435.725125:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233435.852980:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233435.852989:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233435.852991:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233435.852993:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233435.853022:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233435.853026:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233435.853031:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5904 00000100:00080000:1.0:1761233435.853034:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233435.853036:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853038:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233435.853039:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853041:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233435.853043:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233435.853045:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233435.853047:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233435.853049:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233435.853051:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233435.853053:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233435.853055:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853056:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233435.853057:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853058:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233435.853059:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233435.853061:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853063:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233435.853065:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233435.853067:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233435.853068:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853070:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233435.853072:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233435.853078:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761233435.853081:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233435.853084:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233435.853084:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853086:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233435.853089:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233435.853092:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233435.853098:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233435.853103:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233435.853108:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245203904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233435.853116:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233435.853117:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233435.853120:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245203904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233435.853135:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853136:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233435.853138:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233435.853140:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233435.853141:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233435.853142:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233435.853143:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233435.853144:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233435.853146:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5904 00000100:00080000:1.0:1761233435.853147:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233435.853148:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853149:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233435.853150:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853151:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233435.853152:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233435.853153:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233435.853153:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233435.853154:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233435.853155:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233435.853156:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233435.853158:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853159:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233435.853160:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853161:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233435.853162:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233435.853163:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853165:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233435.853166:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233435.853171:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233435.853172:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853174:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233435.853176:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233435.853179:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853181:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233435.853183:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761233435.853184:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233435.853186:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233435.853187:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853189:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233435.853190:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233435.853192:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233435.853197:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233435.853200:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233435.853205:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245203968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233435.853220:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233435.853221:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233435.853225:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245203968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233435.853253:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853255:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233435.853258:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233435.853261:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233435.853263:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233435.853265:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233435.853268:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233435.853270:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233435.853272:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5904 00000100:00080000:1.0:1761233435.853274:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233435.853278:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853281:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233435.853283:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853286:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233435.853288:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233435.853290:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233435.853291:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233435.853292:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233435.853293:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233435.853296:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233435.853302:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853304:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233435.853305:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853306:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233435.853307:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233435.853309:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233435.853313:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233435.853315:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233435.853317:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233435.853318:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853320:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233435.853321:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233435.853324:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853325:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233435.853328:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000020:00000040:1.0:1761233435.853330:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233435.853334:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233435.853334:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853335:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233435.853336:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233435.853337:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233435.853340:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233435.853344:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233435.853349:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245204032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233435.853357:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233435.853358:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233435.853364:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245204032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233435.853375:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233435.853376:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233435.853380:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5914) 00000800:00000200:2.1:1761233435.853466:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233435.853469:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233435.853775:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233435.853777:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233435.854106:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233435.854108:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1761233437.773277:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233437.773283:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233437.773289:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233438.797220:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233438.797242:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233438.797248:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233439.821253:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233439.821258:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233439.821262:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233440.845252:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233440.845257:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233440.845262:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233440.973016:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233440.973042:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233440.973044:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233440.973048:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233440.973051:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233440.973057:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233440.973065:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5909 00000100:00080000:1.0:1761233440.973069:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233440.973073:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973076:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233440.973079:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973081:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233440.973086:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233440.973088:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233440.973091:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233440.973096:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233440.973099:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233440.973102:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233440.973105:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973107:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233440.973109:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973111:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233440.973113:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233440.973117:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973120:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233440.973123:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233440.973127:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233440.973130:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973132:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233440.973134:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233440.973146:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761233440.973149:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233440.973154:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233440.973156:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973159:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233440.973162:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233440.973167:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233440.973176:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233440.973183:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233440.973191:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245204096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233440.973204:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233440.973206:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233440.973210:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245204096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233440.973248:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973253:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233440.973257:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233440.973263:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233440.973265:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233440.973267:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233440.973269:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233440.973285:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233440.973289:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5909 00000100:00080000:1.0:1761233440.973293:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233440.973298:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973301:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233440.973304:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973307:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233440.973311:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233440.973313:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233440.973316:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233440.973320:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233440.973324:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233440.973329:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233440.973336:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973340:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233440.973356:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973361:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233440.973366:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233440.973375:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973384:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233440.973390:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233440.973399:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233440.973408:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973415:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233440.973420:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233440.973430:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973435:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233440.973445:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761233440.973450:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233440.973457:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233440.973461:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973467:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233440.973470:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233440.973473:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233440.973483:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233440.973491:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233440.973504:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245204160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233440.973521:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233440.973525:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233440.973531:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245204160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233440.973547:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973553:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233440.973558:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233440.973565:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233440.973569:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233440.973573:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233440.973575:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233440.973583:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233440.973589:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5909 00000100:00080000:1.0:1761233440.973595:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233440.973605:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973612:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233440.973619:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973627:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233440.973633:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233440.973635:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233440.973639:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233440.973643:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233440.973647:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233440.973649:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233440.973652:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973654:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233440.973656:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973658:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233440.973660:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233440.973664:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233440.973668:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233440.973671:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233440.973676:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233440.973682:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973688:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233440.973692:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233440.973699:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973705:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233440.973712:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761233440.973720:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233440.973727:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233440.973730:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973733:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233440.973736:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233440.973740:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233440.973748:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233440.973758:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233440.973770:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245204224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233440.973787:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233440.973791:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233440.973802:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245204224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:2.1:1761233440.973822:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:1.0:1761233440.973823:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233440.973829:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:2.1:1761233440.973832:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000100:00000040:1.0:1761233440.973836:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5920) 00000800:00000200:2.1:1761233440.974365:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233440.974368:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761233440.974618:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233440.974620:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1761233441.869242:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233441.869246:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233441.869250:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233442.893137:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233442.893141:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233442.893145:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233443.917019:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233443.917022:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233443.917024:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233444.941006:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233444.941009:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233444.941012:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233445.965123:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233445.965126:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233445.965129:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233446.093038:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233446.093049:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233446.093051:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233446.093055:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233446.093058:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233446.093063:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233446.093070:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5915 00000100:00080000:0.0:1761233446.093073:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233446.093076:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093079:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233446.093092:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093094:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233446.093098:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233446.093100:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233446.093103:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233446.093106:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233446.093108:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233446.093110:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233446.093113:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093115:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233446.093116:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093118:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233446.093120:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233446.093123:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093126:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233446.093128:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233446.093132:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233446.093134:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093136:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233446.093138:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233446.093158:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000006edc786f. 00000020:00000040:0.0:1761233446.093163:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233446.093167:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233446.093169:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093172:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233446.093175:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233446.093178:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233446.093186:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233446.093193:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233446.093200:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006edc786f x1846784245204288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233446.093210:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233446.093212:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233446.093215:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000006edc786f] to pc [ptlrpcd_rcv+-1] req@000000006edc786f x1846784245204288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233446.093239:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093242:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233446.093243:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233446.093247:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233446.093248:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233446.093249:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233446.093250:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233446.093253:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233446.093255:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5915 00000100:00080000:0.0:1761233446.093257:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233446.093259:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093260:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233446.093262:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093263:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233446.093265:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233446.093266:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233446.093267:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233446.093269:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233446.093271:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233446.093273:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233446.093275:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093277:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233446.093279:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093280:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233446.093282:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233446.093284:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093286:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233446.093287:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233446.093293:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233446.093294:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093296:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233446.093298:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233446.093301:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093303:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233446.093306:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ad5ae63. 00000020:00000040:0.0:1761233446.093308:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233446.093310:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233446.093310:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093312:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233446.093314:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233446.093316:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233446.093321:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233446.093325:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233446.093329:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ad5ae63 x1846784245204352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233446.093336:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233446.093338:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233446.093341:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ad5ae63] to pc [ptlrpcd_rcv+-1] req@000000009ad5ae63 x1846784245204352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233446.093349:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093353:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233446.093356:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233446.093362:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233446.093364:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233446.093366:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233446.093367:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233446.093371:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233446.093377:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5915 00000100:00080000:0.0:1761233446.093381:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233446.093387:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093392:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233446.093396:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093400:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233446.093403:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233446.093406:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233446.093408:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233446.093412:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233446.093417:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233446.093422:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233446.093429:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093432:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233446.093438:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093442:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233446.093445:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233446.093448:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233446.093454:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233446.093457:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233446.093461:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233446.093463:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093467:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233446.093470:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233446.093473:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093477:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233446.093482:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761233446.093488:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233446.093493:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233446.093496:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093500:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233446.093502:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233446.093504:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233446.093511:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233446.093525:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233446.093533:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245204416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233446.093547:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233446.093550:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233446.093557:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245204416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233446.093568:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233446.093570:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233446.093574:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5925) 00000800:00000200:2.1:1761233446.093648:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233446.093651:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761233446.093982:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233446.093984:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761233446.094272:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233446.094275:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1761233446.989209:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233446.989214:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233446.989218:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233448.013147:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233448.013151:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233448.013154:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233449.037149:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233449.037153:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233449.037156:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233450.061210:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233450.061217:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233450.061224:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233451.085115:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233451.085121:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233451.085127:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233451.213051:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233451.213085:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233451.213088:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233451.213092:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233451.213096:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233451.213104:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233451.213114:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5920 00000100:00080000:1.0:1761233451.213118:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233451.213122:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213125:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233451.213128:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213131:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233451.213135:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233451.213138:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233451.213141:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233451.213146:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233451.213148:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233451.213151:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233451.213154:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213156:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233451.213158:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213160:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233451.213163:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233451.213167:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213169:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233451.213173:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233451.213177:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233451.213180:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213182:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233451.213185:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233451.213200:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f8a85602. 00000020:00000040:1.0:1761233451.213206:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233451.213213:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233451.213214:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213219:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233451.213222:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233451.213228:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233451.213240:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233451.213249:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233451.213259:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f8a85602 x1846784245204480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233451.213271:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233451.213273:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233451.213278:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f8a85602] to pc [ptlrpcd_rcv+-1] req@00000000f8a85602 x1846784245204480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233451.213306:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213308:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233451.213311:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233451.213315:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233451.213316:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233451.213317:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233451.213319:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233451.213322:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233451.213324:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5920 00000100:00080000:1.0:1761233451.213326:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233451.213328:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213330:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233451.213331:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213333:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233451.213335:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233451.213336:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233451.213337:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233451.213339:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233451.213342:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233451.213345:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233451.213348:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213350:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233451.213351:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213353:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233451.213355:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233451.213357:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213359:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233451.213360:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233451.213367:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233451.213369:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213371:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233451.213375:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233451.213378:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213380:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233451.213386:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761233451.213388:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233451.213391:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233451.213392:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213395:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233451.213398:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233451.213400:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233451.213403:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233451.213407:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233451.213416:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245204544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233451.213426:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233451.213428:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233451.213432:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245204544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233451.213439:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213442:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233451.213445:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233451.213452:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233451.213454:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233451.213457:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233451.213463:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233451.213472:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233451.213481:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5920 00000100:00080000:1.0:1761233451.213483:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233451.213492:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213500:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233451.213507:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213514:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233451.213520:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233451.213522:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233451.213526:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233451.213530:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233451.213537:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233451.213562:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233451.213577:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213585:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233451.213607:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213611:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233451.213617:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233451.213621:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233451.213632:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233451.213639:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233451.213643:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233451.213647:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213652:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233451.213655:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233451.213658:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213685:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233451.213696:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761233451.213702:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233451.213709:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233451.213714:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213723:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233451.213725:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233451.213728:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233451.213736:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233451.213744:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233451.213787:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245204608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233451.213818:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233451.213820:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233451.213830:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245204608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233451.213847:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233451.213849:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233451.213898:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5930) 00000800:00000200:2.1:1761233451.213994:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233451.213997:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233451.214292:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233451.214294:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233451.214561:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233451.214563:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1761233452.109079:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233452.109085:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233452.109092:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233453.133122:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233453.133126:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233453.133129:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233454.157151:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233454.157156:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233454.157160:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233455.181193:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233455.181198:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233455.181203:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233456.205032:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233456.205035:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233456.205039:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233456.333658:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233456.333661:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233456.333920:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233456.333921:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233456.334128:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233456.334129:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1761233457.229288:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233457.229292:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233457.229313:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233458.253183:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233458.253188:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233458.253192:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233459.277076:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233459.277079:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233459.277082:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233461.325097:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233461.325102:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233461.325107:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233461.453108:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233461.453131:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233461.453134:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233461.453139:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233461.453144:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233461.453154:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233461.453166:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5930 00000100:00080000:1.0:1761233461.453172:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233461.453178:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453183:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233461.453187:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453215:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233461.453222:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233461.453225:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233461.453229:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233461.453235:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233461.453254:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233461.453273:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233461.453276:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453278:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233461.453282:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453283:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233461.453287:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233461.453291:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453294:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233461.453298:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233461.453303:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233461.453306:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453308:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233461.453327:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233461.453341:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761233461.453346:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233461.453352:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233461.453354:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453358:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233461.453362:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233461.453368:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233461.453378:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233461.453386:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233461.453395:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245204864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233461.453406:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233461.453409:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233461.453413:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245204864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233461.453438:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453440:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233461.453442:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233461.453446:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233461.453447:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233461.453448:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233461.453450:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233461.453452:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233461.453455:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5930 00000100:00080000:1.0:1761233461.453458:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233461.453460:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453462:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233461.453464:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453465:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233461.453467:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233461.453468:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233461.453469:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233461.453471:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233461.453472:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233461.453474:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233461.453476:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453477:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233461.453479:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453480:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233461.453482:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233461.453484:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453486:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233461.453488:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233461.453495:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233461.453496:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453498:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233461.453502:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233461.453505:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453506:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233461.453510:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000020:00000040:1.0:1761233461.453512:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233461.453515:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233461.453516:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453518:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233461.453520:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233461.453522:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233461.453525:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233461.453530:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233461.453535:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245204928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233461.453542:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233461.453543:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233461.453546:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245204928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233461.453553:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453558:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233461.453559:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233461.453564:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233461.453568:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233461.453572:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233461.453574:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233461.453577:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233461.453585:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5930 00000100:00080000:1.0:1761233461.453590:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233461.453614:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453617:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233461.453619:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453621:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233461.453630:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233461.453632:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233461.453638:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233461.453640:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233461.453647:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233461.453650:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233461.453665:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453675:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233461.453681:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453686:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233461.453694:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233461.453702:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233461.453705:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233461.453709:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233461.453715:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233461.453720:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453721:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233461.453724:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233461.453732:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453736:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233461.453746:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761233461.453756:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233461.453760:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233461.453762:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453767:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233461.453770:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233461.453775:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233461.453781:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233461.453785:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233461.453798:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245204992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233461.453940:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233461.453942:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233461.453953:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245204992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233461.453975:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233461.453983:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233461.453992:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5940) 00000800:00000200:3.1:1761233461.454058:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233461.454061:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233461.454391:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233461.454393:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233461.454912:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233461.454914:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1761233463.373101:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233463.373110:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233463.373118:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233465.421030:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233465.421032:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233465.421035:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233466.573008:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233466.573020:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233466.573022:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233466.573025:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233466.573027:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233466.573032:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233466.573038:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5935 00000100:00080000:0.0:1761233466.573041:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233466.573044:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573047:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233466.573049:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573051:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233466.573055:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233466.573057:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233466.573059:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233466.573062:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233466.573064:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233466.573067:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233466.573070:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573072:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233466.573073:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573075:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233466.573077:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233466.573080:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573082:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233466.573085:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233466.573088:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233466.573091:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573093:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233466.573094:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233466.573109:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233466.573112:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233466.573116:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233466.573118:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573120:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233466.573123:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233466.573126:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233466.573134:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233466.573141:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233466.573148:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245205056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233466.573155:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233466.573157:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233466.573159:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245205056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233466.573189:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573191:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233466.573192:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233466.573195:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233466.573196:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233466.573197:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233466.573198:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233466.573199:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233466.573200:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5935 00000100:00080000:0.0:1761233466.573202:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233466.573203:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573205:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233466.573206:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573207:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233466.573208:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233466.573209:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233466.573210:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233466.573211:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233466.573213:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233466.573214:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233466.573216:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573217:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233466.573218:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573220:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233466.573221:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233466.573224:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573226:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233466.573227:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233466.573233:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233466.573235:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573238:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233466.573241:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233466.573246:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573249:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233466.573254:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233466.573257:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233466.573260:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233466.573262:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573265:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233466.573266:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233466.573269:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233466.573275:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233466.573280:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233466.573285:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245205120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233466.573293:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233466.573294:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233466.573297:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245205120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233466.573310:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573312:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233466.573313:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233466.573316:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233466.573317:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233466.573320:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233466.573322:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233466.573326:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233466.573330:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5935 00000100:00080000:0.0:1761233466.573332:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233466.573337:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573340:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233466.573343:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573345:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233466.573349:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233466.573351:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233466.573353:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233466.573355:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233466.573357:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233466.573361:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233466.573364:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573365:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233466.573368:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573371:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233466.573373:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233466.573376:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233466.573379:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233466.573382:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233466.573385:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233466.573390:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573394:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233466.573397:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233466.573403:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573407:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233466.573413:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761233466.573418:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233466.573424:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233466.573426:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573430:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233466.573432:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233466.573435:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233466.573441:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000800:00000200:3.1:1761233466.573441:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233466.573444:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1761233466.573447:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233466.573457:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245205184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233466.573471:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233466.573474:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233466.573482:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245205184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233466.573496:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233466.573500:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233466.573505:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5945) 00000800:00000200:3.1:1761233466.573699:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233466.573700:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233466.573882:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233466.573883:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233467.469076:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233467.469079:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233467.469082:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233469.517072:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233469.517076:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233469.517079:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233471.565010:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233471.565013:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233471.565015:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233471.693012:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233471.693021:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233471.693023:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233471.693025:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233471.693027:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233471.693031:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233471.693037:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5940 00000100:00080000:1.0:1761233471.693039:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233471.693041:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693044:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233471.693045:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693046:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233471.693049:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233471.693050:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233471.693052:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233471.693055:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233471.693056:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233471.693058:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233471.693059:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693061:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233471.693062:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693063:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233471.693065:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233471.693068:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693069:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233471.693071:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233471.693073:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233471.693075:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693077:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233471.693078:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233471.693084:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761233471.693087:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233471.693090:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233471.693091:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693093:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233471.693106:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233471.693110:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233471.693116:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233471.693121:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233471.693126:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245205248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233471.693133:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233471.693135:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233471.693137:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245205248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233471.693154:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693156:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233471.693157:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233471.693159:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233471.693160:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233471.693160:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233471.693161:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233471.693163:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233471.693165:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5940 00000100:00080000:1.0:1761233471.693166:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233471.693168:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693169:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233471.693170:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693171:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233471.693172:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233471.693173:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233471.693173:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233471.693175:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233471.693176:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233471.693177:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233471.693179:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693180:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233471.693181:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693182:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233471.693183:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233471.693185:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693186:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233471.693187:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233471.693191:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233471.693193:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693194:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233471.693196:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233471.693198:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693199:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233471.693202:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761233471.693204:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233471.693216:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233471.693219:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693221:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233471.693224:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233471.693225:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233471.693229:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233471.693231:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233471.693237:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245205312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233471.693258:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233471.693260:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233471.693263:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245205312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233471.693270:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693272:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233471.693273:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233471.693278:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233471.693279:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233471.693280:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233471.693281:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233471.693283:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233471.693287:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5940 00000100:00080000:1.0:1761233471.693291:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233471.693293:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693295:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233471.693296:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693297:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233471.693300:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233471.693302:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233471.693304:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233471.693305:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233471.693308:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233471.693309:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233471.693312:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693315:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233471.693318:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693319:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233471.693323:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233471.693325:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233471.693327:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233471.693330:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233471.693332:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233471.693335:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693339:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233471.693341:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233471.693344:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693346:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233471.693350:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761233471.693353:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233471.693356:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233471.693357:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693359:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233471.693360:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233471.693363:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233471.693368:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233471.693371:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233471.693376:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245205376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233471.693384:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233471.693386:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233471.693390:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245205376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233471.693395:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233471.693396:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233471.693398:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5950) 00000800:00000200:3.1:1761233471.693517:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233471.693533:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761233471.693796:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233471.693797:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761233471.693967:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233471.693968:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1761233472.589046:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233472.589050:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233472.589053:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233474.637157:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233474.637163:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233474.637167:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233476.685048:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233476.685051:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233476.685054:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233476.812976:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233476.812987:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233476.812989:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233476.812992:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233476.812994:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233476.812999:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233476.813006:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5945 00000100:00080000:1.0:1761233476.813010:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233476.813012:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813015:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233476.813017:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813018:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233476.813022:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233476.813024:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233476.813026:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233476.813029:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233476.813031:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233476.813033:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233476.813035:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813036:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233476.813038:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813040:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233476.813042:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233476.813045:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813048:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233476.813051:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233476.813054:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233476.813056:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813058:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233476.813059:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233476.813068:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761233476.813071:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233476.813074:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233476.813076:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813078:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233476.813081:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233476.813084:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233476.813092:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233476.813097:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233476.813104:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245205440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233476.813113:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233476.813115:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233476.813119:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245205440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233476.813141:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813143:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233476.813144:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233476.813147:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233476.813148:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233476.813149:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233476.813150:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233476.813152:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233476.813154:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5945 00000100:00080000:1.0:1761233476.813156:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233476.813158:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813159:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233476.813161:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813163:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233476.813165:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233476.813166:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233476.813167:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233476.813168:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233476.813169:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233476.813171:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233476.813173:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813174:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233476.813175:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813177:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233476.813178:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233476.813180:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813181:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233476.813182:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233476.813187:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233476.813189:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813191:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233476.813193:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233476.813196:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813198:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233476.813201:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000020:00000040:1.0:1761233476.813203:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233476.813205:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233476.813206:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813207:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233476.813208:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233476.813209:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233476.813212:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233476.813215:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233476.813219:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245205504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233476.813225:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233476.813228:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233476.813232:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245205504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233476.813244:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813249:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233476.813252:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233476.813263:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233476.813264:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233476.813268:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233476.813271:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233476.813277:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233476.813285:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5945 00000100:00080000:1.0:1761233476.813291:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233476.813300:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813306:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233476.813313:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813318:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233476.813324:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233476.813327:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233476.813331:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233476.813334:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233476.813336:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233476.813338:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233476.813343:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813348:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233476.813384:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813395:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233476.813399:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233476.813402:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233476.813404:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233476.813409:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233476.813413:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233476.813415:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813418:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233476.813420:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233476.813425:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813431:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233476.813437:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761233476.813441:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233476.813447:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233476.813449:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813453:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233476.813454:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233476.813456:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233476.813461:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233476.813467:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233476.813475:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245205568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233476.813487:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233476.813489:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233476.813496:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245205568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233476.813507:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233476.813509:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233476.813513:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5955) 00000800:00000200:3.1:1761233476.813612:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233476.813616:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761233476.814086:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233476.814089:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761233476.814485:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233476.814488:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1761233478.733084:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233478.733089:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233478.733092:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233481.933021:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233481.933036:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233481.933040:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233481.933044:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233481.933048:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233481.933055:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233481.933062:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5950 00000100:00080000:1.0:1761233481.933066:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233481.933069:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933072:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233481.933075:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933077:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233481.933081:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233481.933083:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233481.933086:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233481.933090:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233481.933092:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233481.933095:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233481.933098:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933100:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233481.933102:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933104:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233481.933107:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233481.933110:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933113:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233481.933116:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233481.933119:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233481.933122:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933124:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233481.933126:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233481.933137:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761233481.933141:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233481.933145:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233481.933147:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933150:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233481.933153:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233481.933158:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233481.933166:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233481.933174:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233481.933181:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245205632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233481.933192:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233481.933195:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233481.933199:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245205632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233481.933228:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933231:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233481.933233:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233481.933237:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233481.933238:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233481.933240:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233481.933242:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233481.933246:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233481.933250:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5950 00000100:00080000:1.0:1761233481.933254:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233481.933258:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933262:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233481.933265:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933293:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233481.933297:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233481.933299:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233481.933302:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233481.933307:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233481.933311:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233481.933315:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233481.933321:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933327:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233481.933336:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933342:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233481.933349:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233481.933361:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933370:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233481.933380:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233481.933393:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233481.933400:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933406:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233481.933415:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233481.933421:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933425:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233481.933439:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761233481.933444:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233481.933447:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233481.933450:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933463:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233481.933473:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233481.933477:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233481.933493:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233481.933501:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233481.933514:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245205696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233481.933547:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233481.933550:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233481.933564:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245205696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233481.933595:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933601:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233481.933604:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233481.933608:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233481.933609:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233481.933611:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233481.933613:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233481.933618:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233481.933624:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5950 00000100:00080000:1.0:1761233481.933636:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233481.933648:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933659:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233481.933667:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933675:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233481.933685:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233481.933688:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233481.933691:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233481.933693:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233481.933699:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233481.933703:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233481.933716:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933726:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233481.933728:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933732:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233481.933735:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233481.933741:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233481.933746:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233481.933750:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233481.933754:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233481.933758:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933762:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233481.933765:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233481.933769:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933771:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233481.933776:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761233481.933784:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233481.933788:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233481.933789:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933790:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233481.933792:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233481.933797:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233481.933803:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233481.933809:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233481.933828:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245205760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233481.933883:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233481.933885:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233481.933890:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245205760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.1:1761233481.933896:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233481.933907:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000100:00000001:1.0:1761233481.933917:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233481.933921:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233481.933926:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5961) 00000800:00000200:3.1:1761233481.934231:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233481.934232:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761233481.934400:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233481.934402:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000100:00080000:1.0:1761233487.053017:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233487.053029:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233487.053031:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233487.053033:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233487.053036:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233487.053041:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233487.053047:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5956 00000100:00080000:1.0:1761233487.053051:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233487.053055:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053058:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233487.053060:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053063:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233487.053067:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233487.053070:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233487.053073:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233487.053076:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233487.053078:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233487.053081:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233487.053084:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053086:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233487.053089:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053091:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233487.053094:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233487.053097:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053100:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233487.053102:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233487.053105:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233487.053108:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053110:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233487.053112:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233487.053120:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761233487.053123:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233487.053127:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233487.053129:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053131:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233487.053134:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233487.053137:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233487.053144:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233487.053150:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233487.053158:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245205824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233487.053168:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233487.053170:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233487.053173:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245205824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233487.053229:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053232:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233487.053234:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233487.053244:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233487.053246:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233487.053247:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233487.053249:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233487.053257:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233487.053259:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5956 00000100:00080000:1.0:1761233487.053261:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233487.053264:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053267:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233487.053270:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053271:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233487.053272:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233487.053273:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233487.053274:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233487.053275:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233487.053276:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233487.053280:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233487.053286:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053288:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233487.053289:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053291:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233487.053294:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233487.053298:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053299:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233487.053300:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233487.053305:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233487.053309:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053315:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233487.053318:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233487.053324:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053327:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233487.053335:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761233487.053339:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233487.053344:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233487.053346:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053351:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233487.053353:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233487.053354:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233487.053358:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233487.053362:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233487.053371:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245205888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233487.053388:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233487.053390:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233487.053394:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245205888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233487.053401:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053407:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233487.053409:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233487.053412:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233487.053413:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233487.053414:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233487.053415:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233487.053416:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233487.053418:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5956 00000100:00080000:1.0:1761233487.053419:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233487.053422:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053425:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233487.053429:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053432:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233487.053435:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233487.053436:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233487.053436:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233487.053438:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233487.053439:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233487.053440:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233487.053443:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053446:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233487.053448:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053449:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233487.053450:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233487.053451:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233487.053453:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233487.053461:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233487.053465:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233487.053469:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053472:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233487.053474:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233487.053478:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053485:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233487.053491:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761233487.053495:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233487.053498:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233487.053500:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053502:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233487.053503:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233487.053503:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233487.053508:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233487.053513:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233487.053521:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245205952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233487.053543:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233487.053548:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233487.053554:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245205952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233487.053574:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233487.053576:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233487.053579:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5966) 00000800:00000200:3.1:1761233487.053656:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233487.053808:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233487.054007:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00080000:0.0:1761233492.172980:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233492.172992:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233492.172994:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233492.172997:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233492.173000:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233492.173005:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233492.173012:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5961 00000100:00080000:0.0:1761233492.173015:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233492.173018:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173021:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233492.173023:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173025:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233492.173029:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233492.173031:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233492.173033:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233492.173036:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233492.173038:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233492.173041:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233492.173043:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173045:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233492.173047:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173048:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233492.173052:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233492.173055:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173057:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233492.173060:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233492.173062:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233492.173065:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173067:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233492.173069:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233492.173074:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233492.173077:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233492.173081:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233492.173083:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173086:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233492.173088:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233492.173092:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233492.173099:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233492.173107:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233492.173129:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245206016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233492.173140:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233492.173142:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233492.173146:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245206016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233492.173170:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173173:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233492.173175:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233492.173179:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233492.173180:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233492.173181:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233492.173182:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233492.173184:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233492.173187:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5961 00000100:00080000:0.0:1761233492.173189:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233492.173191:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173193:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233492.173195:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173197:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233492.173199:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233492.173200:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233492.173201:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233492.173202:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233492.173204:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233492.173206:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233492.173209:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173210:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233492.173217:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173219:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233492.173222:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233492.173226:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173228:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233492.173230:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233492.173236:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233492.173239:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173243:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233492.173247:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233492.173251:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173255:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233492.173259:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233492.173264:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233492.173268:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233492.173270:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173273:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233492.173275:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233492.173279:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233492.173285:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233492.173291:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233492.173301:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245206080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233492.173310:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233492.173312:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233492.173315:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245206080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233492.173326:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173329:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233492.173331:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233492.173338:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233492.173340:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233492.173342:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233492.173343:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233492.173345:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233492.173348:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5961 00000100:00080000:0.0:1761233492.173351:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233492.173356:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173358:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233492.173362:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173366:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233492.173370:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233492.173370:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233492.173372:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233492.173374:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233492.173377:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233492.173381:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233492.173386:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173389:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233492.173395:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173399:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233492.173402:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233492.173408:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233492.173415:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233492.173417:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233492.173421:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233492.173426:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173429:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233492.173432:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233492.173435:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173439:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233492.173445:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233492.173450:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233492.173454:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233492.173454:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173456:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233492.173457:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233492.173459:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233492.173462:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233492.173466:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233492.173472:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245206144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233492.173483:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233492.173486:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233492.173491:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245206144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233492.173511:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233492.173514:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233492.173519:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5971) 00000800:00000200:3.1:1761233492.173556:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233492.173559:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233492.173912:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233492.173914:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233492.174293:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233492.174318:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00080000:1.0:1761233497.293002:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233497.293014:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233497.293016:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233497.293019:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233497.293021:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233497.293025:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233497.293040:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5966 00000100:00080000:1.0:1761233497.293043:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233497.293045:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293047:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233497.293050:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293052:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233497.293056:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233497.293058:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233497.293060:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233497.293063:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233497.293065:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233497.293068:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233497.293070:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293072:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233497.293073:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293075:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233497.293077:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233497.293080:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293082:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233497.293086:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233497.293089:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233497.293091:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293093:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233497.293095:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233497.293104:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761233497.293108:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233497.293112:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233497.293113:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293116:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233497.293119:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233497.293122:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233497.293130:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233497.293136:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233497.293143:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245206208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233497.293152:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233497.293154:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233497.293158:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245206208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233497.293178:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293180:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233497.293182:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233497.293185:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233497.293186:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233497.293187:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233497.293188:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233497.293191:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233497.293192:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5966 00000100:00080000:1.0:1761233497.293194:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233497.293197:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293199:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233497.293201:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293204:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233497.293206:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233497.293207:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233497.293209:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233497.293211:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233497.293213:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233497.293216:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233497.293220:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293224:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233497.293227:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293231:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233497.293236:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233497.293241:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293246:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233497.293250:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233497.293256:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233497.293261:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293265:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233497.293269:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233497.293272:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293274:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233497.293278:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761233497.293282:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233497.293286:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233497.293287:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293288:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233497.293290:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233497.293292:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233497.293298:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233497.293302:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233497.293309:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245206272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233497.293353:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233497.293355:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233497.293359:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245206272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233497.293369:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293371:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233497.293372:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233497.293377:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233497.293378:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233497.293379:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233497.293380:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233497.293382:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233497.293384:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5966 00000100:00080000:1.0:1761233497.293385:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233497.293391:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293393:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233497.293394:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293396:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233497.293399:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233497.293401:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233497.293403:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233497.293406:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233497.293408:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233497.293412:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233497.293418:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293421:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233497.293424:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293427:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233497.293431:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233497.293435:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233497.293437:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233497.293441:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233497.293443:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233497.293446:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293450:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233497.293451:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233497.293452:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293455:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233497.293461:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000020:00000040:1.0:1761233497.293464:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233497.293467:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000800:00000200:3.1:1761233497.293467:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000001:1.0:1761233497.293469:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761233497.293471:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1761233497.293472:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233497.293473:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233497.293474:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233497.293478:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233497.293481:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233497.293485:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245206336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233497.293492:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233497.293493:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233497.293498:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245206336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233497.293506:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233497.293508:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233497.293512:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5976) 00000800:00000200:3.1:1761233497.293726:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233497.293727:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233497.293892:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233497.293893:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1761233501.261142:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233501.261147:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233501.261152:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233502.413024:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233502.413035:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233502.413037:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233502.413040:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233502.413043:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233502.413047:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233502.413054:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5971 00000100:00080000:1.0:1761233502.413056:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233502.413059:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413061:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233502.413063:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413065:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233502.413069:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233502.413070:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233502.413072:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233502.413075:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233502.413077:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233502.413079:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233502.413081:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413083:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233502.413084:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413085:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233502.413087:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233502.413090:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413091:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233502.413093:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233502.413097:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233502.413099:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413101:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233502.413103:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233502.413113:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761233502.413117:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233502.413123:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233502.413124:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413126:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233502.413131:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233502.413136:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233502.413145:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233502.413153:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233502.413161:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245206400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233502.413169:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233502.413171:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233502.413175:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245206400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233502.413195:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413197:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233502.413198:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233502.413201:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233502.413202:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233502.413203:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233502.413204:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233502.413206:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233502.413207:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5971 00000100:00080000:1.0:1761233502.413209:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233502.413210:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413212:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233502.413213:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413214:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233502.413216:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233502.413217:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233502.413218:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233502.413220:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233502.413221:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233502.413222:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233502.413224:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413226:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233502.413227:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413228:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233502.413230:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233502.413232:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413233:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233502.413234:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233502.413239:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233502.413241:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413243:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233502.413246:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233502.413249:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413251:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233502.413253:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761233502.413255:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233502.413259:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233502.413261:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413262:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233502.413263:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233502.413265:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233502.413268:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233502.413270:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233502.413274:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245206464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233502.413280:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233502.413281:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233502.413283:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245206464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233502.413290:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413293:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233502.413295:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233502.413299:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233502.413300:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233502.413302:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233502.413307:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233502.413310:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233502.413314:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5971 00000100:00080000:1.0:1761233502.413318:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233502.413320:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413323:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233502.413325:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413326:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233502.413328:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233502.413329:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233502.413330:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233502.413332:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233502.413335:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233502.413341:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233502.413346:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413350:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233502.413357:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413362:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233502.413368:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233502.413375:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233502.413386:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233502.413388:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233502.413390:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233502.413398:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413402:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233502.413404:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233502.413410:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413412:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233502.413421:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761233502.413425:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233502.413427:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233502.413430:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413432:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233502.413434:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233502.413438:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233502.413443:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233502.413446:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233502.413453:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245206528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233502.413467:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233502.413470:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233502.413473:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245206528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233502.413483:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233502.413492:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233502.413497:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5981) 00000800:00000200:3.1:1761233502.413785:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233502.413791:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1761233502.414407:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233502.414410:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1761233502.415120:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233502.415124:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1761233503.309193:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233503.309198:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233503.309201:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233505.357224:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233505.357243:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233505.357246:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233507.405164:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233507.405167:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233507.405171:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233507.533041:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233507.533053:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233507.533056:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233507.533060:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233507.533062:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233507.533068:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233507.533081:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5976 00000100:00080000:0.0:1761233507.533085:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233507.533089:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533093:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233507.533096:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533098:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233507.533102:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233507.533104:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233507.533106:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233507.533109:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233507.533112:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233507.533114:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233507.533117:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533118:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233507.533120:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533122:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233507.533136:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233507.533139:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533141:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233507.533144:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233507.533147:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233507.533150:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533152:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233507.533154:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233507.533162:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233507.533166:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233507.533170:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233507.533171:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533174:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233507.533177:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233507.533181:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233507.533190:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233507.533206:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233507.533213:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245206592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233507.533223:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233507.533225:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233507.533229:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245206592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233507.533250:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533252:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233507.533255:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233507.533258:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233507.533259:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233507.533261:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233507.533262:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233507.533264:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233507.533267:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5976 00000100:00080000:0.0:1761233507.533269:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233507.533271:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533273:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233507.533275:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533277:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233507.533279:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233507.533280:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233507.533281:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233507.533283:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233507.533286:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233507.533289:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233507.533292:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533294:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233507.533296:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533298:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233507.533300:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233507.533304:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533306:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233507.533308:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233507.533314:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233507.533317:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533319:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233507.533321:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233507.533324:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533326:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233507.533331:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233507.533340:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233507.533345:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233507.533348:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533351:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233507.533355:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233507.533357:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233507.533361:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233507.533368:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233507.533378:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245206656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233507.533402:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233507.533404:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233507.533411:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245206656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233507.533437:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533442:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233507.533445:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233507.533452:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233507.533457:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233507.533460:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233507.533462:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233507.533476:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233507.533483:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5976 00000100:00080000:0.0:1761233507.533485:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233507.533488:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533491:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233507.533499:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533506:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233507.533510:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233507.533514:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233507.533517:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233507.533522:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233507.533526:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233507.533528:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233507.533534:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533540:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233507.533543:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533547:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233507.533558:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233507.533568:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233507.533574:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233507.533576:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233507.533583:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233507.533590:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533598:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233507.533601:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233507.533610:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533624:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233507.533628:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233507.533634:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233507.533642:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233507.533645:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533647:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233507.533650:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233507.533654:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233507.533661:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233507.533666:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233507.533674:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245206720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233507.533689:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233507.533691:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233507.533694:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245206720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233507.533702:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233507.533706:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233507.533711:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5986) 00000800:00000200:3.1:1761233507.533793:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233507.533798:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1761233507.534227:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233507.534229:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1761233507.534464:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233507.534466:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1761233509.453109:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233509.453114:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233509.453118:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233511.501118:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233511.501122:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233511.501126:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233512.652997:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233512.653010:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233512.653011:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233512.653014:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233512.653017:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233512.653023:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233512.653031:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5981 00000100:00080000:0.0:1761233512.653035:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233512.653038:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653042:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233512.653044:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653046:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233512.653050:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233512.653052:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233512.653054:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233512.653057:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233512.653061:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233512.653064:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233512.653068:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653070:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233512.653072:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653073:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233512.653076:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233512.653081:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653083:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233512.653087:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233512.653089:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233512.653092:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653094:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233512.653096:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233512.653105:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233512.653108:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233512.653111:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233512.653112:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653115:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233512.653118:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233512.653121:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233512.653128:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233512.653133:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233512.653139:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245206784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233512.653150:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233512.653152:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233512.653157:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245206784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233512.653183:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653186:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233512.653188:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233512.653192:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233512.653194:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233512.653195:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233512.653196:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233512.653200:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233512.653202:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5981 00000100:00080000:0.0:1761233512.653205:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233512.653207:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653209:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233512.653211:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653213:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233512.653215:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233512.653216:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233512.653217:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233512.653219:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233512.653221:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233512.653223:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233512.653226:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653229:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233512.653231:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653233:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233512.653236:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233512.653239:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653241:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233512.653244:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233512.653250:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233512.653253:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653255:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233512.653258:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233512.653263:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653265:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233512.653268:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233512.653272:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233512.653279:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233512.653281:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653287:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233512.653291:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233512.653294:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233512.653299:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233512.653306:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233512.653321:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245206848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233512.653339:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233512.653343:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233512.653352:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245206848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233512.653377:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653382:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233512.653389:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233512.653395:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233512.653396:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233512.653398:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233512.653402:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233512.653409:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233512.653416:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5981 00000100:00080000:0.0:1761233512.653419:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233512.653426:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653428:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233512.653433:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653437:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233512.653441:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233512.653444:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233512.653450:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233512.653454:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233512.653460:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233512.653466:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233512.653476:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653482:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233512.653489:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653496:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233512.653503:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233512.653513:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233512.653519:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233512.653523:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233512.653528:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233512.653533:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653534:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233512.653536:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233512.653538:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653541:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233512.653545:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233512.653549:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233512.653557:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233512.653558:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653560:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233512.653562:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233512.653565:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233512.653573:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233512.653579:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233512.653589:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245206912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233512.653613:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233512.653616:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233512.653626:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245206912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233512.653645:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233512.653647:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233512.653650:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5991) 00000800:00000200:3.1:1761233512.653677:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233512.653681:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1761233512.654133:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233512.654135:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1761233512.654382:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233512.654402:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1761233513.549049:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233513.549052:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233513.549054:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233514.573006:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233514.573009:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233514.573012:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233515.597051:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233515.597055:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233515.597058:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233516.621081:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233516.621085:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233516.621089:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233517.645077:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233517.645080:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233517.645082:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233517.773031:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233517.773043:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233517.773045:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233517.773047:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233517.773050:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233517.773055:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233517.773061:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5986 00000100:00080000:1.0:1761233517.773064:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233517.773067:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773071:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233517.773073:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773075:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233517.773079:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233517.773081:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233517.773084:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233517.773086:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233517.773089:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233517.773091:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233517.773094:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773095:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233517.773098:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773100:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233517.773102:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233517.773116:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773119:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233517.773122:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233517.773125:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233517.773128:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773130:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233517.773133:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233517.773141:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761233517.773144:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233517.773149:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233517.773150:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773153:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233517.773155:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233517.773159:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233517.773168:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233517.773174:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233517.773180:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245206976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233517.773187:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233517.773189:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233517.773191:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245206976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233517.773206:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773208:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233517.773209:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233517.773211:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233517.773212:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233517.773213:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233517.773214:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233517.773216:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233517.773218:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5986 00000100:00080000:1.0:1761233517.773219:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233517.773221:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773222:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233517.773223:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773224:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233517.773226:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233517.773226:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233517.773227:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233517.773228:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233517.773229:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233517.773230:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233517.773232:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773233:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233517.773233:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773234:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233517.773236:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233517.773245:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773247:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233517.773249:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233517.773254:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233517.773260:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773262:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233517.773268:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233517.773273:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773275:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233517.773281:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761233517.773284:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233517.773288:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233517.773290:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773293:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233517.773296:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233517.773299:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233517.773305:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233517.773311:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233517.773318:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245207040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233517.773324:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233517.773326:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233517.773330:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245207040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233517.773337:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773340:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233517.773341:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233517.773343:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233517.773345:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233517.773347:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233517.773348:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233517.773351:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233517.773354:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5986 00000100:00080000:1.0:1761233517.773357:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233517.773360:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773362:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233517.773365:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773369:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233517.773372:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233517.773373:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233517.773375:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233517.773377:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233517.773381:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233517.773384:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233517.773389:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773394:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233517.773396:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773398:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233517.773401:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233517.773404:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233517.773405:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233517.773406:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233517.773409:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233517.773410:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773411:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233517.773412:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233517.773415:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773417:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233517.773420:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000020:00000040:1.0:1761233517.773423:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233517.773426:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233517.773427:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773431:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233517.773433:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233517.773435:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233517.773439:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233517.773442:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233517.773448:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245207104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233517.773461:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233517.773463:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233517.773467:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245207104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233517.773475:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233517.773477:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233517.773481:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (5996) 00000800:00000200:3.1:1761233517.773490:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233517.773492:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1761233517.773721:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233517.773723:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1761233517.773903:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233517.773905:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000020:00000001:1.0:1761233518.669122:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233518.669128:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233518.669134:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233519.693140:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233519.693145:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233519.693149:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233520.717045:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233520.717049:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233520.717052:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233521.741046:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233521.741052:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233521.741056:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233522.765202:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233522.765220:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233522.765226:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233522.893036:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233522.893049:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233522.893051:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233522.893054:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233522.893056:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233522.893062:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233522.893069:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5991 00000100:00080000:0.0:1761233522.893072:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233522.893075:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893079:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233522.893081:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893083:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233522.893087:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233522.893089:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233522.893092:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233522.893096:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233522.893098:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233522.893100:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233522.893103:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893105:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233522.893107:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893109:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233522.893112:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233522.893116:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893119:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233522.893123:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233522.893126:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233522.893130:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893132:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233522.893134:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233522.893145:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000095969832. 00000020:00000040:0.0:1761233522.893148:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233522.893153:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233522.893154:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893157:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233522.893160:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233522.893163:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233522.893173:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233522.893180:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233522.893187:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000095969832 x1846784245207168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233522.893198:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233522.893201:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233522.893204:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000095969832] to pc [ptlrpcd_rcv+-1] req@0000000095969832 x1846784245207168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233522.893231:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893234:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233522.893236:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233522.893241:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233522.893242:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233522.893243:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233522.893245:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233522.893248:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233522.893250:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5991 00000100:00080000:0.0:1761233522.893268:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233522.893272:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893274:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233522.893276:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893279:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233522.893282:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233522.893283:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233522.893285:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233522.893287:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233522.893289:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233522.893292:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233522.893295:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893299:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233522.893303:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893306:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233522.893310:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233522.893316:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893320:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233522.893322:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233522.893328:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233522.893333:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893355:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233522.893358:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233522.893369:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893374:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233522.893382:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002ca01b78. 00000020:00000040:0.0:1761233522.893388:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233522.893395:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233522.893397:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893401:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233522.893403:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233522.893406:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233522.893413:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233522.893419:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233522.893426:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ca01b78 x1846784245207232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233522.893445:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233522.893448:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233522.893466:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002ca01b78] to pc [ptlrpcd_rcv+-1] req@000000002ca01b78 x1846784245207232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233522.893475:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893478:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233522.893479:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233522.893483:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233522.893485:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233522.893487:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233522.893489:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233522.893492:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233522.893500:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5991 00000100:00080000:0.0:1761233522.893506:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233522.893515:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893519:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233522.893544:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893552:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233522.893558:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233522.893562:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233522.893565:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233522.893567:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233522.893573:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233522.893576:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233522.893578:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893581:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233522.893584:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893606:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233522.893609:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233522.893612:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233522.893614:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233522.893615:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233522.893618:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233522.893620:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893622:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233522.893623:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233522.893645:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893646:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233522.893650:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233522.893652:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233522.893670:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233522.893671:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893672:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233522.893673:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233522.893675:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233522.893677:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233522.893680:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233522.893688:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245207296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233522.893703:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233522.893704:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233522.893705:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245207296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233522.893714:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233522.893718:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233522.893722:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6002) 00000800:00000200:3.1:1761233522.893799:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233522.893803:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233522.894150:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233522.894152:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233522.894394:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233522.894396:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1761233523.789070:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233523.789075:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233523.789081:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233524.813079:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233524.813082:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233524.813085:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233525.837054:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233525.837057:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233525.837060:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233526.861064:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233526.861067:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233526.861070:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233527.885059:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233527.885062:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233527.885064:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233528.012973:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233528.012982:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233528.012984:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233528.012986:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233528.012988:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233528.012992:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233528.012998:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 5997 00000100:00080000:0.0:1761233528.013001:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233528.013003:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013021:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233528.013023:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013025:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233528.013028:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233528.013029:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233528.013031:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233528.013034:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233528.013035:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233528.013037:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233528.013039:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013040:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233528.013041:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013042:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233528.013044:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233528.013047:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013048:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233528.013050:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233528.013053:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233528.013054:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013056:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233528.013057:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233528.013065:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233528.013067:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233528.013071:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233528.013072:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013074:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233528.013076:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233528.013079:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233528.013085:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233528.013091:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233528.013097:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245207360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233528.013104:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233528.013105:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233528.013108:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245207360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233528.013126:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013127:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233528.013128:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233528.013131:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233528.013131:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233528.013132:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233528.013133:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233528.013135:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233528.013136:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5997 00000100:00080000:0.0:1761233528.013137:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233528.013139:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013140:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233528.013141:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013142:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233528.013143:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233528.013144:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233528.013145:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233528.013146:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233528.013147:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233528.013148:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233528.013150:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013151:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233528.013152:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013154:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233528.013156:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233528.013157:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013159:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233528.013161:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233528.013165:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233528.013166:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013167:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233528.013170:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233528.013172:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013175:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233528.013178:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233528.013182:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233528.013184:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233528.013185:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013187:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233528.013189:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233528.013190:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233528.013194:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233528.013198:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233528.013203:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245207424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233528.013210:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233528.013212:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233528.013219:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245207424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233528.013227:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013230:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233528.013232:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233528.013234:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233528.013236:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233528.013239:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233528.013242:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233528.013243:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233528.013246:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 5997 00000100:00080000:0.0:1761233528.013249:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233528.013251:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013253:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233528.013256:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013258:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233528.013260:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233528.013262:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233528.013265:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233528.013286:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233528.013289:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233528.013292:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233528.013296:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013299:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233528.013302:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013304:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233528.013308:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233528.013311:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233528.013317:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233528.013321:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233528.013324:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233528.013326:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013329:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233528.013330:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233528.013334:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013337:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233528.013341:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233528.013342:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233528.013343:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233528.013344:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013345:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233528.013347:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233528.013348:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233528.013352:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233528.013355:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233528.013360:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245207488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233528.013370:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233528.013372:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233528.013378:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245207488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233528.013384:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233528.013386:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233528.013390:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6007) 00000800:00000200:3.1:1761233528.013416:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233528.013418:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233528.013705:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233528.013708:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233528.013928:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233528.013929:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1761233528.909044:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233528.909047:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233528.909049:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233529.932990:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233529.932993:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233529.932995:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233530.957043:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233530.957046:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233530.957049:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233531.981033:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233531.981037:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233531.981039:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233533.005128:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233533.005131:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233533.005133:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233533.132958:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233533.132969:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233533.132972:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233533.132975:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233533.132978:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233533.132983:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233533.132989:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6002 00000100:00080000:1.0:1761233533.132992:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233533.132995:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.132998:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233533.133000:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133002:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233533.133006:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233533.133008:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233533.133010:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233533.133013:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233533.133016:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233533.133018:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233533.133021:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133023:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233533.133024:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133026:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233533.133029:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233533.133032:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133035:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233533.133039:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233533.133042:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233533.133045:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133047:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233533.133049:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233533.133057:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761233533.133061:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233533.133065:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233533.133066:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133069:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233533.133070:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233533.133074:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233533.133080:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233533.133085:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233533.133090:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245207552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233533.133097:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233533.133098:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233533.133101:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245207552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233533.133116:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133117:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233533.133119:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233533.133121:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233533.133122:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233533.133123:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233533.133124:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233533.133126:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233533.133127:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6002 00000100:00080000:1.0:1761233533.133128:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233533.133129:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133130:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233533.133131:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133133:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233533.133135:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233533.133136:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233533.133137:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233533.133138:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233533.133140:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233533.133142:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233533.133143:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133145:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233533.133145:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133147:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233533.133148:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233533.133150:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133152:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233533.133154:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233533.133160:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233533.133162:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133165:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233533.133167:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233533.133171:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133174:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233533.133178:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761233533.133181:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233533.133184:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233533.133186:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133187:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233533.133190:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233533.133191:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233533.133195:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233533.133198:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233533.133205:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245207616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233533.133220:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233533.133221:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233533.133224:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245207616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233533.133233:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133235:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233533.133245:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233533.133249:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233533.133250:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233533.133251:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233533.133253:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233533.133258:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233533.133262:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6002 00000100:00080000:1.0:1761233533.133263:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233533.133267:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133268:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233533.133269:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133270:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233533.133272:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233533.133274:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233533.133275:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233533.133276:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233533.133279:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233533.133282:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233533.133287:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133289:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233533.133292:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133295:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233533.133298:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233533.133302:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233533.133304:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233533.133307:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233533.133309:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233533.133312:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133314:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233533.133316:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233533.133319:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133322:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233533.133325:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761233533.133328:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233533.133331:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233533.133332:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133334:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233533.133335:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233533.133336:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233533.133338:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233533.133340:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233533.133342:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245207680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233533.133347:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233533.133348:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233533.133352:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245207680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233533.133358:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233533.133359:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233533.133362:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6012) 00000800:00000200:3.1:1761233533.133412:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233533.133415:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233533.133709:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233533.133710:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233533.133872:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233533.133886:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233534.029033:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233534.029036:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233534.029038:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233535.053053:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233535.053056:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233535.053059:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233536.076986:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233536.076992:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233536.076997:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233537.101122:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233537.101126:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233537.101130:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233538.125048:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233538.125051:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233538.125053:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233538.252968:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233538.252979:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233538.252981:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233538.252983:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233538.252985:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233538.252989:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233538.252994:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6007 00000100:00080000:1.0:1761233538.252997:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233538.252999:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253001:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233538.253002:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253004:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233538.253006:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233538.253008:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233538.253010:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233538.253012:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233538.253014:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233538.253016:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233538.253017:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253019:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233538.253020:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253021:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233538.253023:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233538.253025:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253027:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233538.253029:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233538.253031:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233538.253033:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253035:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233538.253036:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233538.253045:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f8a85602. 00000020:00000040:1.0:1761233538.253047:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233538.253050:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233538.253051:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253053:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233538.253055:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233538.253058:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233538.253066:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233538.253071:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233538.253076:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f8a85602 x1846784245207744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233538.253082:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233538.253084:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233538.253087:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f8a85602] to pc [ptlrpcd_rcv+-1] req@00000000f8a85602 x1846784245207744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233538.253104:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253105:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233538.253107:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233538.253109:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233538.253110:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233538.253111:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233538.253112:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233538.253113:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233538.253114:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6007 00000100:00080000:1.0:1761233538.253115:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233538.253117:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253119:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233538.253120:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253121:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233538.253122:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233538.253123:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233538.253124:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233538.253125:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233538.253126:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233538.253127:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233538.253129:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253130:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233538.253131:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253132:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233538.253133:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233538.253135:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253136:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233538.253137:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233538.253142:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233538.253143:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253144:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233538.253146:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233538.253148:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253149:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233538.253152:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761233538.253154:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233538.253156:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233538.253156:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253158:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233538.253160:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233538.253161:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233538.253164:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233538.253167:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233538.253172:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245207808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233538.253200:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233538.253201:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233538.253205:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245207808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233538.253213:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253215:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233538.253216:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233538.253219:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233538.253220:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233538.253222:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233538.253223:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233538.253226:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233538.253229:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6007 00000100:00080000:1.0:1761233538.253231:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233538.253233:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253234:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233538.253235:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253237:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233538.253239:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233538.253240:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233538.253241:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233538.253242:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233538.253244:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233538.253246:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233538.253250:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253254:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233538.253255:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253258:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233538.253261:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233538.253264:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233538.253269:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233538.253272:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233538.253276:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233538.253280:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253282:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233538.253284:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233538.253289:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253290:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233538.253295:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761233538.253299:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233538.253303:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233538.253305:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253308:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233538.253310:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233538.253312:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233538.253317:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233538.253322:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233538.253331:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245207872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233538.253344:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233538.253346:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233538.253352:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245207872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233538.253360:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233538.253362:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233538.253367:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6017) 00000800:00000200:3.1:1761233538.253441:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233538.253444:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761233538.253674:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233538.253676:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761233538.253806:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233538.253807:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1761233539.149036:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233539.149041:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233539.149045:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233540.173086:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233540.173090:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233540.173093:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233540.812991:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233540.812996:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233540.812997:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233540.813000:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233540.813005:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233540.813008:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233540.813010:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233540.813011:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233540.813013:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233541.836952:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233541.836956:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233541.836958:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233541.836961:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233541.836965:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233541.836967:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233541.836969:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233541.836971:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233541.836973:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233542.221070:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233542.221073:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233542.221076:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233542.860994:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233542.860998:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233542.860999:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233542.861003:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233542.861007:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233542.861010:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233542.861012:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233542.861014:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233542.861015:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761233543.372986:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233543.372996:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233543.372997:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233543.372999:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233543.373001:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233543.373005:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233543.373011:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6012 00000100:00080000:1.0:1761233543.373014:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233543.373016:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373018:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233543.373020:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373021:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233543.373024:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233543.373026:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233543.373028:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233543.373031:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233543.373033:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233543.373035:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233543.373038:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373039:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233543.373040:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373042:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233543.373043:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233543.373045:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373047:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233543.373049:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233543.373051:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233543.373054:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373056:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233543.373058:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233543.373065:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761233543.373068:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233543.373072:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233543.373073:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373076:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233543.373078:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233543.373081:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233543.373088:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233543.373093:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233543.373100:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245207936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233543.373107:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233543.373109:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233543.373112:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245207936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233543.373125:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373126:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233543.373127:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233543.373130:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233543.373130:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233543.373131:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233543.373132:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233543.373134:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233543.373136:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6012 00000100:00080000:1.0:1761233543.373137:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233543.373138:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373139:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233543.373140:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373141:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233543.373143:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233543.373144:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233543.373144:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233543.373146:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233543.373147:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233543.373149:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233543.373151:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373152:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233543.373153:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373155:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233543.373156:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233543.373158:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373159:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233543.373160:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233543.373165:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233543.373168:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373170:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233543.373173:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233543.373177:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373179:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233543.373182:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000020:00000040:1.0:1761233543.373185:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233543.373188:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233543.373189:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373192:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233543.373193:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233543.373194:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233543.373197:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233543.373202:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233543.373207:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245208000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233543.373217:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233543.373220:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233543.373226:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245208000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233543.373234:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373236:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233543.373238:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233543.373242:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233543.373244:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233543.373246:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233543.373247:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233543.373250:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233543.373252:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6012 00000100:00080000:1.0:1761233543.373253:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233543.373258:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373260:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233543.373261:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373266:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233543.373269:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233543.373270:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233543.373273:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233543.373274:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233543.373276:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233543.373280:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233543.373286:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373289:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233543.373292:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373295:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233543.373298:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233543.373317:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233543.373319:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233543.373321:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233543.373323:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233543.373324:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373325:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233543.373327:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233543.373330:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373334:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233543.373337:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761233543.373341:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233543.373344:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233543.373345:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373346:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233543.373347:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233543.373348:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233543.373351:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233543.373353:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233543.373358:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245208064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233543.373365:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233543.373366:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233543.373371:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245208064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233543.373383:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233543.373387:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233543.373390:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6022) 00000800:00000200:3.1:1761233543.373422:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233543.373424:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761233543.373677:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233543.373678:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761233543.373813:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233543.373814:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000100:00000001:0.0:1761233543.884944:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233543.884952:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233543.884954:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233543.884959:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233543.884964:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233543.884967:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233543.884970:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233543.884972:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233543.884975:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233544.269035:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233544.269038:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233544.269041:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233544.908982:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233544.908988:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233544.908989:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233544.908994:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233544.908999:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233544.909002:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233544.909003:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233544.909005:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233544.909007:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233545.932921:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233545.932930:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233545.932932:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233545.932939:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233545.932946:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233545.932949:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233545.932951:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233545.932953:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233545.932955:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233546.317181:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233546.317189:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233546.317195:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233546.957031:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233546.957036:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233546.957038:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233546.957042:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233546.957048:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233546.957051:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233546.957054:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233546.957057:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233546.957060:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233547.980963:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233547.980970:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233547.980972:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233547.980978:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233547.980983:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233547.980987:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233547.980989:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233547.980992:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233547.980994:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233548.365148:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233548.365152:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233548.365155:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233548.493649:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233548.493653:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761233548.493998:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233548.493999:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761233548.494160:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233548.494162:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1761233549.004991:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233549.004996:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233549.004998:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233549.005002:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233549.005022:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233549.005025:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233549.005027:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233549.005029:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233549.005031:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233551.053031:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233551.053036:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233551.053037:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233551.053041:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233551.053045:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233551.053048:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233551.053050:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233551.053052:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233551.053054:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233552.077029:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233552.077036:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233552.077038:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233552.077045:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233552.077051:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233552.077057:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233552.077059:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233552.077063:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233552.077066:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233553.100968:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233553.100974:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233553.100976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233553.100982:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233553.100989:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233553.100993:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233553.100997:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233553.100999:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233553.101002:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761233553.613807:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233553.613991:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000800:00000200:3.1:1761233553.614009:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233553.614067:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:0.0:1761233553.614110:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000800:00000200:3.1:1761233553.614114:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:0.0:1761233554.124940:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233554.124949:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233554.124951:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233554.124957:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233554.124965:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233554.124969:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233554.124972:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233554.124975:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233554.124978:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233555.149019:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233555.149025:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233555.149027:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233555.149031:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233555.149037:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233555.149040:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233555.149042:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233555.149045:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233555.149047:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233556.172959:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233556.172965:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233556.172967:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233556.172972:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233556.172977:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233556.172981:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233556.172984:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233556.172987:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233556.172990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:1.0:1761233556.685187:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:1.0:1761233556.685233:0:88985:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1761233556.685242:0:88985:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:1.0:1761233556.685254:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:1.0:1761233556.685257:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676778). 00000400:00000200:1.0:1761233556.685264:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1761233556.685268:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1761233556.685274:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000038d726e9 00000400:00000010:1.0:1761233556.685276:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000038d726e9. 00000100:00000001:1.0:1761233556.685281:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233556.685287:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000007d4d64bb x1846784245208320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233578 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233556.685311:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233556.685314:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d4d64bb x1846784245208320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233578 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233556.685321:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233556.685322:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:1.0:1761233556.685333:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:1.0:1761233556.685335:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d4888078 (tot 1676530). 00000400:00000200:1.0:1761233556.685338:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1761233556.685341:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1761233556.685344:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000e8fa2ea2 00000400:00000010:1.0:1761233556.685346:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000e8fa2ea2. 00000100:00000001:1.0:1761233556.685349:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233556.685352:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000007d82b1f3 x1846784245208384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233578 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233556.685361:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233556.685365:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d82b1f3 x1846784245208384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233578 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233556.685372:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233556.685374:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:1.0:1761233556.685383:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:1.0:1761233556.685389:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:1.0:1761233556.685395:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1761233556.685400:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1761233556.685413:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a049cc4d 00000400:00000010:1.0:1761233556.685418:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a049cc4d. 00000100:00000001:1.0:1761233556.685423:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233556.685430:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@0000000069aeaefe x1846784245208448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233578 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233556.685460:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233556.685468:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000069aeaefe x1846784245208448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233578 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233556.685480:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233556.685482:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:0.0:1761233557.196969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233557.196975:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233557.196977:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233557.196982:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233557.196988:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233557.196992:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233557.196995:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233557.196997:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233557.197000:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.221046:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233558.221055:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233558.221058:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233558.221066:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233558.221114:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233558.221121:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233558.221124:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233558.221128:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233558.221132:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1761233558.733129:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233558.733197:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233558.733200:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233558.733204:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233558.733208:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233558.733216:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233558.733227:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6027 00000100:00080000:0.0:1761233558.733231:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233558.733238:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733243:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233558.733247:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733250:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233558.733256:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233558.733259:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233558.733267:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233558.733272:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233558.733276:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233558.733279:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233558.733283:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733285:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233558.733287:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733290:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233558.733293:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233558.733297:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733301:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233558.733305:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233558.733310:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233558.733314:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733317:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233558.733319:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233558.733329:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233558.733335:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233558.733341:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233558.733343:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733349:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233558.733353:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233558.733360:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233558.733375:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233558.733387:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233558.733398:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245208512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233558.733411:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233558.733415:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233558.733420:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245208512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233558.733455:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733458:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233558.733461:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233558.733465:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233558.733466:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233558.733468:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233558.733469:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233558.733473:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233558.733476:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6027 00000100:00080000:0.0:1761233558.733479:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233558.733483:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733486:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233558.733488:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733491:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233558.733494:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233558.733497:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233558.733499:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233558.733502:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233558.733505:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233558.733507:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233558.733512:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733517:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233558.733522:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733527:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233558.733532:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233558.733541:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733547:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233558.733551:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233558.733561:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233558.733567:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733572:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233558.733580:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233558.733587:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733592:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233558.733601:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233558.733605:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233558.733609:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233558.733610:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733614:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233558.733618:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233558.733620:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233558.733628:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233558.733636:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233558.733643:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245208576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233558.733655:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233558.733658:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233558.733665:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245208576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233558.733680:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733685:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233558.733687:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233558.733692:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233558.733695:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233558.733697:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233558.733700:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233558.733706:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233558.733714:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6027 00000100:00080000:0.0:1761233558.733719:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233558.733723:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733729:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233558.733734:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733738:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233558.733742:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233558.733744:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233558.733745:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233558.733748:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233558.733751:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233558.733753:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233558.733760:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733764:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233558.733769:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733771:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233558.733774:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233558.733781:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233558.733787:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233558.733790:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233558.733795:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233558.733802:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733806:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233558.733810:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233558.733816:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733818:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233558.733824:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233558.733829:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233558.733835:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233558.733838:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733841:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233558.733914:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233558.733918:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233558.733926:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233558.733932:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233558.733942:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245208640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233558.733957:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000800:00000200:3.1:1761233558.733957:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:0.0:1761233558.733961:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761233558.733965:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00000040:0.0:1761233558.733970:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245208640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233558.733982:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233558.733987:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233558.733995:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6037) 00000800:00000200:3.1:1761233558.734605:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233558.734636:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233558.734997:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233558.735000:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1761233559.244977:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233559.244983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233559.244986:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233559.244990:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233559.244996:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233559.245000:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233559.245003:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233559.245006:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233559.245008:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233559.629294:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233559.629301:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233559.629307:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233560.268965:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233560.268969:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233560.268971:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233560.268974:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233560.268978:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233560.268981:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233560.268983:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233560.268985:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233560.268987:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233561.292960:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233561.292965:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233561.292967:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233561.292972:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233561.292978:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233561.292981:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233561.292983:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233561.292986:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233561.292989:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233561.677062:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233561.677066:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233561.677069:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233562.317029:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233562.317036:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233562.317038:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233562.317043:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233562.317049:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233562.317053:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233562.317056:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233562.317058:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233562.317060:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233563.725122:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233563.725126:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233563.725130:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233563.853689:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233563.853691:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233563.853975:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233563.853976:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233563.854165:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233563.854166:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1761233565.773080:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233565.773083:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233565.773086:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233567.821185:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233567.821191:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233567.821198:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233568.973572:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233568.973576:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1761233568.974009:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233568.974012:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1761233568.974253:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233568.974256:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000020:00000001:1.0:1761233569.869074:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233569.869078:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233569.869081:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233571.533051:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233571.533058:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233571.533060:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233571.533064:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233571.533078:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233571.533084:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233571.533087:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233571.533089:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233571.533091:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233571.917062:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233571.917065:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233571.917067:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233572.556985:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233572.556991:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233572.556993:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233572.556998:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233572.557004:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233572.557008:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233572.557010:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233572.557013:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233572.557015:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233572.941039:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233572.941043:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233572.941047:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233573.581050:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233573.581058:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233573.581060:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233573.581066:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233573.581071:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233573.581076:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233573.581078:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233573.581081:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233573.581084:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233573.965119:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233573.965129:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233573.965154:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233574.093524:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233574.093530:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761233574.094012:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233574.094015:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761233574.094231:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233574.094233:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000100:00000001:0.0:1761233574.604955:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233574.604960:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233574.604961:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233574.604965:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233574.604969:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233574.604972:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233574.604974:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233574.604976:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233574.604978:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233574.989039:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233574.989043:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233574.989047:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233575.629042:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233575.629050:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233575.629052:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233575.629057:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233575.629062:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233575.629067:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233575.629071:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233575.629074:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233575.629077:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233576.013092:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233576.013096:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233576.013099:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233576.652973:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233576.652980:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233576.652983:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233576.652990:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233576.652997:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233576.653001:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233576.653005:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233576.653008:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233576.653012:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233577.037075:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233577.037080:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233577.037084:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233577.677012:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233577.677020:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233577.677024:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233577.677031:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233577.677038:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233577.677042:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233577.677048:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233577.677051:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233577.677054:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233578.061162:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233578.061167:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233578.061170:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233578.700962:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233578.700968:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233578.700971:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233578.700976:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233578.700982:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233578.700987:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233578.700990:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233578.700993:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233578.700997:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233579.085116:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233579.085120:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233579.085124:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233579.213770:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233579.213777:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1761233579.214291:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233579.214294:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1761233579.214558:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233579.214560:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000100:00000001:0.0:1761233579.725017:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233579.725022:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233579.725024:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233579.725029:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233579.725034:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233579.725037:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233579.725040:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233579.725042:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233579.725044:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233580.109039:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233580.109042:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233580.109045:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233580.749014:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233580.749022:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233580.749024:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233580.749031:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233580.749039:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233580.749043:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233580.749046:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233580.749050:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233580.749053:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233581.133165:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233581.133171:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233581.133176:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233581.772985:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233581.772992:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233581.772993:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233581.772998:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233581.773003:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233581.773006:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233581.773008:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233581.773010:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233581.773013:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233582.157090:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233582.157094:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233582.157099:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233582.797013:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233582.797022:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233582.797025:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233582.797032:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233582.797041:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233582.797046:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233582.797051:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233582.797054:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233582.797059:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233583.181188:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233583.181192:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233583.181196:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233583.821062:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233583.821068:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233583.821070:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233583.821074:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233583.821079:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233583.821082:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233583.821084:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233583.821087:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233583.821089:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233584.205195:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233584.205201:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233584.205205:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233584.333468:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233584.333471:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761233584.333683:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233584.333685:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761233584.333940:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233584.333941:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1761233584.845167:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233584.845175:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233584.845178:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233584.845184:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233584.845193:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233584.845197:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233584.845201:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233584.845204:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233584.845207:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233585.229212:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233585.229218:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233585.229226:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233585.868996:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233585.869000:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233585.869001:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233585.869004:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233585.869008:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233585.869011:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233585.869013:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233585.869014:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233585.869016:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233586.253057:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233586.253060:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233586.253062:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233586.893052:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233586.893060:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233586.893062:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233586.893069:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233586.893075:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233586.893079:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233586.893082:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233586.893085:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233586.893088:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233587.277137:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233587.277142:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233587.277147:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233587.917010:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233587.917017:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233587.917019:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233587.917026:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233587.917032:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233587.917037:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233587.917040:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233587.917043:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233587.917046:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233588.301022:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233588.301025:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233588.301028:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233588.941072:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233588.941078:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233588.941080:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233588.941086:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233588.941091:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233588.941096:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233588.941099:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233588.941102:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233588.941104:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233589.325222:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233589.325228:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233589.325234:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233589.452964:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233589.452973:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233589.452975:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233589.452977:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233589.452979:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233589.452984:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233589.452990:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6058 00000100:00080000:0.0:1761233589.452992:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233589.452994:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.452996:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233589.452998:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.452999:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233589.453002:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233589.453004:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233589.453006:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233589.453008:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233589.453009:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233589.453011:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233589.453013:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453015:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233589.453016:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453017:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233589.453019:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233589.453022:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453023:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233589.453025:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233589.453028:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233589.453030:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453031:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233589.453033:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233589.453040:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233589.453043:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233589.453046:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233589.453047:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453049:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233589.453052:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233589.453054:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233589.453061:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233589.453066:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233589.453071:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245209664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233589.453079:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233589.453081:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233589.453083:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245209664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233589.453100:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453101:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233589.453102:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233589.453105:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233589.453106:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233589.453107:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233589.453108:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233589.453110:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233589.453111:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6058 00000100:00080000:0.0:1761233589.453113:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233589.453115:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453116:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233589.453117:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453118:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233589.453119:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233589.453121:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233589.453122:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233589.453123:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233589.453124:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233589.453126:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233589.453128:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453129:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233589.453130:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453132:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233589.453133:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233589.453135:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453137:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233589.453140:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233589.453145:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233589.453148:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453151:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233589.453154:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233589.453158:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453161:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233589.453166:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233589.453169:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233589.453173:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233589.453174:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453177:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233589.453179:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233589.453183:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233589.453189:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233589.453196:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233589.453204:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245209728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233589.453218:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233589.453221:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233589.453226:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245209728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233589.453243:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453247:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233589.453249:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233589.453259:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233589.453262:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233589.453266:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233589.453268:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233589.453274:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233589.453280:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6058 00000100:00080000:0.0:1761233589.453282:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233589.453286:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453291:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233589.453296:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453300:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233589.453305:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233589.453308:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233589.453312:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233589.453314:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233589.453318:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233589.453321:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233589.453326:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453329:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233589.453334:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453337:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233589.453338:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233589.453343:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233589.453346:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233589.453348:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233589.453352:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233589.453354:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453356:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233589.453357:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233589.453361:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453363:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233589.453367:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761233589.453370:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233589.453375:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233589.453376:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453378:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233589.453379:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.1:1761233589.453380:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000001:0.0:1761233589.453381:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761233589.453386:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 02000000:00000010:0.0:1761233589.453387:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233589.453391:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233589.453397:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245209792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233589.453407:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233589.453408:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233589.453411:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245209792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233589.453425:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233589.453428:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233589.453433:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6068) 00000800:00000200:3.1:1761233589.453694:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233589.453696:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233589.453835:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233589.453837:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000100:00000001:0.0:1761233589.964994:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233589.965001:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233589.965003:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233589.965017:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233589.965023:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233589.965027:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233589.965031:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233589.965034:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233589.965038:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233590.349025:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233590.349029:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233590.349032:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233590.989037:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233590.989041:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233590.989042:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233590.989047:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233590.989051:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233590.989054:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233590.989056:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233590.989058:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233590.989060:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233591.373152:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233591.373159:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233591.373165:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233592.013002:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233592.013012:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233592.013016:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233592.013024:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233592.013033:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233592.013039:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233592.013044:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233592.013047:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233592.013053:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233592.397106:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233592.397109:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233592.397112:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233593.036965:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233593.036970:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233593.036971:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233593.036974:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233593.036978:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233593.036981:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233593.036983:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233593.036985:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233593.036987:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233593.421148:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233593.421152:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233593.421156:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233594.060919:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233594.060924:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233594.060926:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233594.060929:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233594.060933:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233594.060936:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233594.060937:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233594.060939:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233594.060940:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233594.445056:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233594.445059:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233594.445061:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233594.573034:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233594.573044:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233594.573046:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233594.573048:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233594.573051:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233594.573055:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233594.573062:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6063 00000100:00080000:1.0:1761233594.573066:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233594.573068:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573070:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233594.573072:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573074:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233594.573076:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233594.573078:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233594.573080:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233594.573083:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233594.573085:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233594.573086:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233594.573088:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573090:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233594.573091:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573092:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233594.573094:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233594.573097:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573099:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233594.573101:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233594.573103:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233594.573105:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573123:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233594.573125:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233594.573134:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761233594.573137:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233594.573140:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233594.573153:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573155:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233594.573158:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233594.573162:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233594.573171:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233594.573179:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233594.573195:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245209856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233594.573207:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233594.573210:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233594.573215:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245209856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233594.573242:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573245:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233594.573247:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233594.573252:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233594.573253:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233594.573255:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233594.573256:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233594.573259:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233594.573262:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6063 00000100:00080000:1.0:1761233594.573265:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233594.573267:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573269:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233594.573270:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573271:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233594.573273:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233594.573274:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233594.573275:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233594.573276:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233594.573278:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233594.573280:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233594.573282:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573284:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233594.573286:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573287:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233594.573289:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233594.573292:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573294:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233594.573296:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233594.573302:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233594.573303:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573305:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233594.573308:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233594.573312:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573313:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233594.573317:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761233594.573318:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233594.573321:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233594.573321:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573326:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233594.573329:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233594.573333:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233594.573338:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233594.573346:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233594.573351:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245209920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233594.573403:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233594.573406:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233594.573415:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245209920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233594.573427:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573430:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233594.573433:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233594.573448:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233594.573455:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233594.573457:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233594.573460:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233594.573471:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233594.573481:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6063 00000100:00080000:1.0:1761233594.573484:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233594.573487:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573493:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233594.573520:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573543:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233594.573546:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233594.573547:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233594.573549:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233594.573553:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233594.573557:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233594.573560:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233594.573568:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573574:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233594.573588:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573599:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233594.573604:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233594.573611:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233594.573619:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233594.573625:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233594.573628:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233594.573631:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573634:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233594.573636:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233594.573642:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573647:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233594.573652:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761233594.573655:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233594.573658:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233594.573660:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573663:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233594.573667:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233594.573671:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233594.573678:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233594.573683:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233594.573689:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245209984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233594.573701:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233594.573702:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233594.573707:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245209984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233594.573711:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233594.573713:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233594.573718:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6073) 00000800:00000200:3.1:1761233594.573806:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233594.573809:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233594.574206:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233594.574208:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233594.574394:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233594.574395:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1761233595.084990:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233595.084994:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233595.084995:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233595.084999:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233595.085003:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233595.085005:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233595.085007:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233595.085009:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233595.085011:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233595.469234:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233595.469239:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233595.469244:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233596.108928:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233596.108932:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233596.108933:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233596.108937:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233596.108941:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233596.108944:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233596.108946:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233596.108947:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233596.108949:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233596.493212:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233596.493235:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233596.493238:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233597.517133:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233597.517139:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233597.517145:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233598.541280:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233598.541285:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233598.541290:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233599.565120:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233599.565124:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233599.565128:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233599.693070:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233599.693089:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233599.693092:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233599.693096:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233599.693100:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233599.693109:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233599.693119:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6068 00000100:00080000:1.0:1761233599.693125:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233599.693130:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693135:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233599.693139:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693143:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233599.693149:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233599.693152:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233599.693157:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233599.693161:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233599.693165:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233599.693169:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233599.693174:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693177:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233599.693180:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693183:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233599.693187:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233599.693191:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693194:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233599.693198:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233599.693202:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233599.693206:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693209:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233599.693211:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233599.693223:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761233599.693227:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233599.693232:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233599.693234:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693238:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233599.693241:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233599.693246:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233599.693256:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233599.693263:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233599.693272:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245210048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233599.693284:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233599.693287:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233599.693292:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245210048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233599.693330:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693333:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233599.693336:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233599.693341:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233599.693342:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233599.693344:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233599.693346:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233599.693349:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233599.693352:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6068 00000100:00080000:1.0:1761233599.693354:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233599.693357:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693359:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233599.693361:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693362:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233599.693365:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233599.693366:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233599.693368:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233599.693370:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233599.693372:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233599.693374:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233599.693377:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693378:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233599.693381:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693383:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233599.693385:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233599.693388:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693391:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233599.693393:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233599.693399:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233599.693401:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693403:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233599.693406:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233599.693410:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693412:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233599.693416:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000020:00000040:1.0:1761233599.693478:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233599.693482:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233599.693484:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693486:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233599.693488:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233599.693490:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233599.693497:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233599.693502:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233599.693508:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245210112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233599.693517:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233599.693526:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233599.693533:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245210112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233599.693547:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693552:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233599.693554:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233599.693562:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233599.693565:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233599.693568:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233599.693571:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233599.693578:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233599.693585:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6068 00000100:00080000:1.0:1761233599.693591:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233599.693598:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693605:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233599.693610:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693615:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233599.693621:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233599.693623:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233599.693628:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233599.693634:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233599.693640:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233599.693645:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233599.693649:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693655:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233599.693660:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693665:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233599.693668:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233599.693677:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233599.693684:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233599.693689:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233599.693694:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233599.693700:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693705:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233599.693708:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233599.693716:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693719:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233599.693726:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761233599.693730:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233599.693737:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233599.693741:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693750:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233599.693752:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233599.693754:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233599.693760:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233599.693768:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233599.693773:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245210176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233599.693787:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233599.693790:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233599.693795:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245210176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233599.693808:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233599.693813:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233599.693819:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6078) 00000800:00000200:3.1:1761233599.694084:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233599.694088:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233599.694508:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233599.694511:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233599.694782:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233599.694784:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233600.589065:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233600.589069:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233600.589072:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233602.637097:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233602.637102:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233602.637107:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233604.685134:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233604.685139:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233604.685143:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233604.812997:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233604.813011:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233604.813012:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233604.813015:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233604.813019:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233604.813025:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233604.813034:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6073 00000100:00080000:1.0:1761233604.813038:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233604.813042:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813046:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233604.813049:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813052:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233604.813057:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233604.813059:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233604.813062:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233604.813066:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233604.813068:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233604.813070:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233604.813072:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813074:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233604.813078:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813080:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233604.813083:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233604.813087:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813089:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233604.813092:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233604.813095:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233604.813098:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813100:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233604.813103:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233604.813116:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761233604.813119:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233604.813125:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233604.813126:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813130:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233604.813134:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233604.813139:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233604.813149:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233604.813156:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233604.813165:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245210240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233604.813180:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233604.813182:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233604.813186:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245210240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233604.813203:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813205:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233604.813206:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233604.813209:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233604.813210:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233604.813212:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233604.813213:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233604.813215:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233604.813217:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6073 00000100:00080000:1.0:1761233604.813218:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233604.813220:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813221:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233604.813222:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813223:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233604.813225:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233604.813225:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233604.813226:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233604.813228:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233604.813229:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233604.813230:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233604.813232:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813233:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233604.813234:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813236:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233604.813237:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233604.813239:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813241:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233604.813242:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233604.813249:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233604.813251:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813253:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233604.813258:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233604.813261:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813263:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233604.813266:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761233604.813268:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233604.813270:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233604.813271:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813272:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233604.813273:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233604.813276:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233604.813282:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233604.813287:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233604.813291:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245210304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233604.813300:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233604.813301:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233604.813306:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245210304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233604.813315:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813318:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233604.813319:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233604.813330:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233604.813331:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233604.813333:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233604.813335:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233604.813355:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233604.813358:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6073 00000100:00080000:1.0:1761233604.813362:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233604.813369:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813375:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233604.813380:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813385:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233604.813391:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233604.813394:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233604.813401:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233604.813406:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233604.813409:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233604.813411:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233604.813416:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813418:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233604.813420:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813425:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233604.813428:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233604.813434:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233604.813441:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233604.813444:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233604.813447:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233604.813450:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813453:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233604.813456:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233604.813458:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813460:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233604.813469:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761233604.813478:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233604.813481:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233604.813482:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813486:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233604.813489:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233604.813492:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233604.813498:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233604.813505:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233604.813509:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245210368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233604.813520:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233604.813522:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233604.813528:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245210368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233604.813535:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233604.813541:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233604.813545:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6083) 00000800:00000200:3.1:1761233604.813686:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233604.813689:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761233604.814139:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233604.814142:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761233604.814470:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233604.814472:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1761233605.709063:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233605.709068:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233605.709074:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233606.733223:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233606.733227:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233606.733231:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233607.757015:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233607.757019:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233607.757022:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233608.781097:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233608.781101:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233608.781104:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233609.805163:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233609.805170:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233609.805177:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233609.933593:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233609.933596:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1761233609.934013:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233609.934014:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1761233609.934187:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233609.934189:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000020:00000001:1.0:1761233610.829181:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233610.829189:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233610.829196:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233611.853315:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233611.853341:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233611.853350:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233612.877655:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233612.877658:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233612.877661:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233613.517055:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233613.517064:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233613.517066:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233613.517072:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233613.517094:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233613.517098:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233613.517101:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233613.517103:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233613.517106:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233613.901113:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233613.901118:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233613.901124:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233614.541054:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233614.541062:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233614.541065:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233614.541071:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233614.541078:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233614.541084:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233614.541088:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233614.541091:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233614.541094:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233614.925243:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233614.925250:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233614.925255:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233615.054098:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233615.054103:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761233615.054654:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233615.054658:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761233615.055028:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233615.055031:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000100:00000001:0.0:1761233615.564945:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233615.564951:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233615.564953:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233615.564959:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233615.564965:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233615.564969:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233615.564971:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233615.564974:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233615.564977:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233615.949095:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233615.949100:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233615.949104:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233616.589082:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233616.589090:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233616.589092:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233616.589098:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233616.589105:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233616.589109:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233616.589134:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233616.589138:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233616.589145:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233616.973162:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233616.973166:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233616.973169:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233617.612995:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233617.613002:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233617.613004:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233617.613010:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233617.613017:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233617.613022:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233617.613026:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233617.613028:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233617.613032:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233617.997205:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233617.997210:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233617.997214:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233619.021049:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233619.021052:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233619.021055:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233620.045251:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233620.045260:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233620.045266:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233620.173605:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233620.173740:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233620.173797:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000800:00000200:3.1:1761233620.173945:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233620.173969:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:0.0:1761233620.174014:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000100:00000001:0.0:1761233620.685023:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233620.685030:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233620.685034:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233620.685040:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233620.685045:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233620.685049:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233620.685051:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233620.685053:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233620.685055:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233621.069047:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233621.069050:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233621.069054:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233621.709154:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233621.709165:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233621.709169:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233621.709178:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233621.709188:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233621.709195:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233621.709200:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233621.709205:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233621.709209:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233622.093121:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233622.093125:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233622.093128:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233622.733025:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233622.733035:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233622.733037:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233622.733044:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233622.733051:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233622.733070:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233622.733074:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233622.733076:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233622.733079:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233623.117040:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233623.117044:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233623.117047:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1761233623.245283:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:0.0:1761233623.245335:0:88983:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1761233623.245344:0:88983:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:0.0:1761233623.245356:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233623.245359:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aaa22236 (tot 1676778). 00000400:00000200:0.0:1761233623.245366:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233623.245371:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233623.245377:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000048b8c7e9 00000400:00000010:0.0:1761233623.245379:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000048b8c7e9. 00000100:00000001:0.0:1761233623.245383:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233623.245392:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000007d4d64bb x1846784245210816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233645 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233623.245422:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233623.245426:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d4d64bb x1846784245210816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233645 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233623.245434:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233623.245437:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233623.245450:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233623.245453:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006315f304 (tot 1676530). 00000400:00000200:0.0:1761233623.245457:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233623.245461:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233623.245465:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000003df587c7 00000400:00000010:0.0:1761233623.245467:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000003df587c7. 00000100:00000001:0.0:1761233623.245470:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233623.245475:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@0000000069aeaefe x1846784245210880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233645 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233623.245500:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233623.245506:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000069aeaefe x1846784245210880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233645 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233623.245516:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233623.245521:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233623.245532:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233623.245537:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:0.0:1761233623.245545:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233623.245570:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233623.245575:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000f750763e 00000400:00000010:0.0:1761233623.245580:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000f750763e. 00000100:00000001:0.0:1761233623.245585:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233623.245594:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000007d82b1f3 x1846784245210944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233645 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233623.245608:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233623.245614:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d82b1f3 x1846784245210944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233645 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233623.245634:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233623.245637:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:0.0:1761233623.756993:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233623.757000:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233623.757013:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233623.757018:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233623.757024:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233623.757029:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233623.757032:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233623.757035:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233623.757038:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233624.141099:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233624.141103:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233624.141107:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233624.780940:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233624.780949:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233624.780952:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233624.780959:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233624.780967:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233624.780972:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233624.780976:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233624.780980:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233624.780983:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233625.165225:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233625.165232:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233625.165240:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233625.293241:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233625.293258:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233625.293261:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233625.293265:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233625.293269:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233625.293279:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233625.293289:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6094 00000100:00080000:0.0:1761233625.293295:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233625.293298:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293302:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233625.293305:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293308:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233625.293313:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233625.293318:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233625.293322:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233625.293326:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233625.293329:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233625.293332:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233625.293336:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293338:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233625.293340:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293341:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233625.293345:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233625.293349:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293352:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233625.293356:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233625.293360:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233625.293364:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.293367:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233625.293369:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233625.293378:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233625.293383:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233625.293389:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233625.293390:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.293395:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233625.293399:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233625.293405:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233625.293416:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233625.293427:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233625.293437:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245211008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233625.293451:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233625.293456:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233625.293463:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245211008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233625.293507:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.293512:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233625.293516:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233625.293522:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233625.293524:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233625.293526:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233625.293528:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233625.293533:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233625.293537:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6094 00000100:00080000:0.0:1761233625.293541:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233625.293546:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293549:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233625.293552:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293555:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233625.293560:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233625.293563:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233625.293566:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233625.293571:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233625.293576:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233625.293580:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233625.293585:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293589:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233625.293593:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293596:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233625.293601:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233625.293610:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293618:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233625.293626:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233625.293637:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233625.293646:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.293654:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233625.293662:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233625.293672:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.293680:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233625.293689:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233625.293701:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233625.293713:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233625.293719:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.293726:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233625.293730:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233625.293738:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233625.293754:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233625.293767:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233625.293782:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245211072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233625.293801:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233625.293807:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233625.293814:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245211072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233625.293891:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.293897:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233625.293900:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233625.293906:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233625.293909:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233625.293912:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233625.293915:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233625.293921:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233625.293927:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6094 00000100:00080000:0.0:1761233625.293931:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233625.293944:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293954:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233625.293962:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.293971:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233625.293981:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233625.293986:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233625.293992:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233625.293997:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233625.294005:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233625.294013:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233625.294068:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.294076:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233625.294080:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233625.294083:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233625.294087:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233625.294093:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233625.294098:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233625.294106:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233625.294115:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233625.294122:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.294131:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233625.294137:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233625.294148:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.294155:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233625.294168:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761233625.294177:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233625.294184:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233625.294187:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.294195:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233625.294201:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233625.294208:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233625.294252:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233625.294264:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761233625.294273:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00100000:0.0:1761233625.294282:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245211136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.1:1761233625.294282:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1761233625.294316:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233625.294321:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233625.294334:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245211136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233625.294367:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233625.294375:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233625.294389:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6104) 00000800:00000200:3.1:1761233625.295242:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233625.295247:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233625.295711:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233625.295716:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1761233625.805047:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233625.805056:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233625.805058:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233625.805066:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233625.805074:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233625.805080:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233625.805084:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233625.805087:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233625.805092:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233626.189148:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233626.189153:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233626.189157:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233626.829070:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233626.829077:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233626.829079:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233626.829086:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233626.829092:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233626.829096:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233626.829099:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233626.829102:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233626.829105:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233627.213102:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233627.213107:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233627.213111:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233627.853099:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233627.853106:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233627.853110:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233627.853116:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233627.853122:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233627.853126:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233627.853130:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233627.853132:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233627.853135:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233628.237149:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233628.237152:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233628.237155:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233628.876990:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233628.876997:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233628.876999:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233628.877005:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233628.877023:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233628.877027:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233628.877029:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233628.877032:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233628.877034:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233629.261331:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233629.261337:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233629.261343:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233629.901007:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233629.901012:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233629.901013:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233629.901017:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233629.901021:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233629.901024:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233629.901026:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233629.901028:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233629.901029:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233630.285310:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233630.285317:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233630.285323:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233630.413082:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233630.413092:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233630.413093:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233630.413096:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233630.413098:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233630.413104:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233630.413110:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6099 00000100:00080000:0.0:1761233630.413113:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233630.413115:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413118:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233630.413120:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413122:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233630.413125:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233630.413127:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233630.413130:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233630.413133:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233630.413135:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233630.413138:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233630.413140:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413144:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233630.413145:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413147:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233630.413151:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233630.413154:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413157:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233630.413160:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233630.413163:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233630.413166:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413168:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233630.413169:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233630.413179:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ad5ae63. 00000020:00000040:0.0:1761233630.413182:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233630.413187:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233630.413188:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413207:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233630.413211:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233630.413216:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233630.413227:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233630.413234:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233630.413242:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ad5ae63 x1846784245211200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233630.413252:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233630.413256:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233630.413260:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ad5ae63] to pc [ptlrpcd_rcv+-1] req@000000009ad5ae63 x1846784245211200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233630.413297:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413299:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233630.413301:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233630.413304:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233630.413305:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233630.413306:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233630.413308:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233630.413310:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233630.413312:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6099 00000100:00080000:0.0:1761233630.413314:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233630.413317:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413318:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233630.413319:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413321:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233630.413322:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233630.413323:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233630.413324:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233630.413326:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233630.413327:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233630.413328:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233630.413330:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413332:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233630.413333:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413335:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233630.413336:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233630.413339:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413341:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233630.413342:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233630.413348:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233630.413350:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413352:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233630.413355:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233630.413359:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413360:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233630.413363:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000006edc786f. 00000020:00000040:0.0:1761233630.413364:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233630.413367:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233630.413370:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413377:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233630.413380:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233630.413383:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233630.413387:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233630.413393:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233630.413399:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006edc786f x1846784245211264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233630.413408:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233630.413409:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233630.413416:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000006edc786f] to pc [ptlrpcd_rcv+-1] req@000000006edc786f x1846784245211264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233630.413496:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413501:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233630.413504:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233630.413508:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233630.413511:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233630.413515:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233630.413518:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233630.413525:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233630.413531:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6099 00000100:00080000:0.0:1761233630.413535:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233630.413538:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413543:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233630.413551:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413558:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233630.413560:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233630.413561:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233630.413562:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233630.413563:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233630.413565:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233630.413567:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233630.413577:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413581:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233630.413586:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413598:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233630.413601:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233630.413604:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233630.413610:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233630.413612:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233630.413615:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233630.413618:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413622:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233630.413626:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233630.413633:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413635:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233630.413643:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761233630.413649:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233630.413658:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233630.413661:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413672:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233630.413675:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233630.413679:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233630.413685:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233630.413690:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233630.413698:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245211328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233630.413706:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233630.413708:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233630.413714:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245211328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233630.413733:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233630.413734:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233630.413738:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6109) 00000800:00000200:3.1:1761233630.414010:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233630.414015:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233630.414427:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233630.414430:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761233630.414793:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233630.414797:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1761233630.924923:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233630.924945:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233630.924949:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233630.924954:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233630.924960:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233630.924963:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233630.924966:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233630.924979:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233630.924982:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233631.309143:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233631.309147:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233631.309151:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233631.948940:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233631.948947:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233631.948949:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233631.948954:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233631.948958:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233631.948962:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233631.948965:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233631.948967:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233631.948970:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233632.333074:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233632.333077:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233632.333081:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233632.972983:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233632.972992:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233632.972994:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233632.973001:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233632.973006:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233632.973010:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233632.973012:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233632.973015:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233632.973017:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233633.357115:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233633.357119:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233633.357123:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233633.996940:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233633.996948:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233633.996950:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233633.996957:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233633.996963:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233633.996968:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233633.996970:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233633.996973:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233633.996976:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233634.381051:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233634.381056:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233634.381060:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233635.405123:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233635.405127:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233635.405131:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233635.533600:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233635.533602:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233635.533913:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233635.533914:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233635.534056:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233635.534058:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1761233636.429135:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233636.429155:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233636.429159:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233637.453222:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233637.453227:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233637.453231:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233638.477125:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233638.477129:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233638.477134:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233639.501184:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233639.501191:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233639.501196:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233640.525063:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233640.525066:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233640.525070:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233640.653035:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233640.653046:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233640.653049:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233640.653052:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233640.653054:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233640.653059:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233640.653065:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6109 00000100:00080000:0.0:1761233640.653068:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233640.653070:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653073:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233640.653075:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653076:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233640.653079:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233640.653080:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233640.653082:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233640.653085:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233640.653087:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233640.653089:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233640.653091:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653092:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233640.653093:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653095:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233640.653097:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233640.653099:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653101:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233640.653103:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233640.653105:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233640.653107:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653109:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233640.653110:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233640.653118:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233640.653121:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233640.653125:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233640.653126:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653129:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233640.653131:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233640.653134:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233640.653141:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233640.653146:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233640.653152:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245211584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233640.653160:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233640.653162:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233640.653165:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245211584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233640.653185:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653187:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233640.653188:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233640.653191:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233640.653192:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233640.653193:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233640.653194:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233640.653196:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233640.653197:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6109 00000100:00080000:0.0:1761233640.653199:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233640.653200:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653201:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233640.653203:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653204:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233640.653205:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233640.653206:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233640.653207:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233640.653208:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233640.653209:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233640.653210:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233640.653213:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653214:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233640.653215:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653217:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233640.653218:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233640.653220:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653222:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233640.653223:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233640.653228:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233640.653229:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653231:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233640.653233:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233640.653236:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653237:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233640.653241:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233640.653244:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233640.653246:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233640.653249:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653251:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233640.653253:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233640.653255:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233640.653259:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233640.653263:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233640.653271:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245211648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233640.653281:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233640.653284:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233640.653294:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245211648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233640.653306:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653309:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233640.653313:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233640.653318:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233640.653318:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233640.653319:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233640.653321:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233640.653326:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233640.653330:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6109 00000100:00080000:0.0:1761233640.653332:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233640.653336:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653338:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233640.653340:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653345:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233640.653346:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233640.653347:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233640.653348:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233640.653350:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233640.653355:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233640.653358:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233640.653363:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653365:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233640.653370:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653373:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233640.653377:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233640.653381:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233640.653385:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233640.653387:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233640.653390:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233640.653394:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653398:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233640.653400:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233640.653402:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653404:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233640.653409:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761233640.653412:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233640.653414:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233640.653414:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653416:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233640.653416:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233640.653418:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233640.653421:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233640.653426:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233640.653432:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245211712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233640.653440:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233640.653448:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233640.653452:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245211712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233640.653465:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233640.653470:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233640.653475:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6119) 00000800:00000200:2.1:1761233640.653508:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233640.653510:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761233640.653764:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233640.653766:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761233640.654100:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233640.654102:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1761233641.549182:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233641.549188:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233641.549195:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233642.573168:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233642.573173:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233642.573177:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233643.597180:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233643.597184:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233643.597189:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233644.621072:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233644.621075:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233644.621078:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233645.645050:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233645.645054:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233645.645056:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233645.772904:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233645.772913:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233645.772915:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233645.772917:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233645.772919:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233645.772923:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233645.772929:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6114 00000100:00080000:0.0:1761233645.772931:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233645.772933:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.772935:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233645.772937:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.772938:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233645.772940:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233645.772943:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233645.772945:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233645.772947:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233645.772949:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233645.772950:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233645.772952:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.772953:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233645.772954:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.772955:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233645.772956:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233645.772959:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233645.772960:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233645.772970:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233645.772973:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233645.772975:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.772981:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233645.772983:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233645.772991:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233645.772994:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233645.772998:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233645.773000:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773002:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233645.773005:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233645.773009:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233645.773016:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233645.773023:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233645.773030:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245211776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233645.773040:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233645.773042:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233645.773046:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245211776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233645.773068:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773071:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233645.773073:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233645.773075:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233645.773076:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233645.773077:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233645.773078:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233645.773079:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233645.773081:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6114 00000100:00080000:0.0:1761233645.773082:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233645.773083:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773085:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233645.773086:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773087:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233645.773089:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233645.773089:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233645.773090:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233645.773091:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233645.773093:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233645.773094:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233645.773096:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773097:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233645.773098:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773099:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233645.773101:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233645.773103:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773105:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233645.773106:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233645.773111:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233645.773113:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773114:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233645.773117:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233645.773120:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773123:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233645.773126:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233645.773129:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233645.773133:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233645.773134:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773135:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233645.773136:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233645.773138:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233645.773143:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233645.773147:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233645.773153:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245211840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233645.773161:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233645.773163:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233645.773167:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245211840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233645.773177:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773180:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233645.773181:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233645.773187:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233645.773188:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233645.773191:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233645.773193:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233645.773195:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233645.773197:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6114 00000100:00080000:0.0:1761233645.773200:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233645.773204:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773206:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233645.773209:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773210:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233645.773212:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233645.773213:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233645.773215:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233645.773216:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233645.773219:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233645.773222:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233645.773223:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773224:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233645.773226:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773228:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233645.773230:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233645.773232:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233645.773233:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233645.773235:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233645.773238:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233645.773240:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773241:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233645.773242:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233645.773244:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773247:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233645.773251:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233645.773255:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233645.773258:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233645.773259:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773260:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233645.773261:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233645.773263:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233645.773268:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233645.773272:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233645.773278:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245211904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233645.773289:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233645.773291:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233645.773295:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245211904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233645.773304:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233645.773305:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233645.773308:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6124) 00000800:00000200:2.1:1761233645.773383:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233645.773386:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761233645.773740:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233645.773742:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761233645.774011:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233645.774013:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1761233646.669058:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233646.669063:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233646.669068:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233647.693112:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233647.693115:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233647.693117:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233648.717100:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233648.717103:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233648.717105:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233649.741126:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233649.741131:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233649.741135:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233650.765037:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233650.765042:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233650.765047:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233650.892927:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233650.892935:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233650.892937:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233650.892939:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233650.892941:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233650.892945:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233650.892951:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6119 00000100:00080000:0.0:1761233650.892954:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233650.892956:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.892958:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233650.892959:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.892961:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233650.892963:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233650.892965:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233650.892967:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233650.892969:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233650.892970:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233650.892972:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233650.892974:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.892975:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233650.892976:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.892977:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233650.892979:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233650.892981:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233650.892982:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233650.892986:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233650.892988:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233650.892990:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.892991:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233650.892993:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233650.892999:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233650.893001:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233650.893005:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233650.893006:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893008:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233650.893011:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233650.893013:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233650.893020:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233650.893025:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233650.893030:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245211968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233650.893037:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233650.893038:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233650.893041:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245211968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233650.893057:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893058:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233650.893060:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233650.893062:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233650.893063:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233650.893064:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233650.893065:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233650.893066:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233650.893068:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6119 00000100:00080000:0.0:1761233650.893069:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233650.893070:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893071:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233650.893072:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893074:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233650.893075:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233650.893076:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233650.893077:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233650.893078:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233650.893079:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233650.893080:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233650.893081:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893082:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233650.893083:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893084:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233650.893085:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233650.893086:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893087:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233650.893088:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233650.893092:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233650.893093:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893094:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233650.893096:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233650.893098:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893099:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233650.893101:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233650.893102:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233650.893103:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233650.893104:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893105:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233650.893105:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233650.893106:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233650.893110:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233650.893112:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233650.893114:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245212032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233650.893122:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233650.893124:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233650.893126:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245212032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233650.893131:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893132:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233650.893133:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233650.893135:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233650.893136:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233650.893137:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233650.893137:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233650.893139:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233650.893141:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6119 00000100:00080000:0.0:1761233650.893143:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233650.893146:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893149:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233650.893152:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893154:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233650.893158:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233650.893159:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233650.893160:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233650.893161:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233650.893164:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233650.893167:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233650.893172:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893176:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233650.893178:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893180:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233650.893182:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233650.893187:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233650.893192:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233650.893194:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233650.893197:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233650.893200:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893203:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233650.893205:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233650.893207:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893209:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233650.893214:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233650.893217:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233650.893218:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233650.893219:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893222:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233650.893224:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233650.893225:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233650.893228:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233650.893231:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233650.893234:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245212096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233650.893242:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233650.893244:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233650.893249:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245212096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233650.893271:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233650.893274:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233650.893278:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6130) 00000800:00000200:2.1:1761233650.893345:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233650.893347:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233650.893511:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233650.893512:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233650.893734:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233650.893735:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1761233651.789036:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233651.789039:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233651.789041:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233652.813117:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233652.813121:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233652.813124:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233653.837111:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233653.837114:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233653.837117:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233654.861077:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233654.861080:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233654.861083:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233655.885080:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233655.885084:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233655.885087:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233656.012996:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233656.013008:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233656.013010:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233656.013012:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233656.013015:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233656.013021:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233656.013028:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6125 00000100:00080000:0.0:1761233656.013031:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233656.013033:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013036:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233656.013038:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013039:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233656.013043:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233656.013044:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233656.013047:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233656.013050:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233656.013052:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233656.013053:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233656.013055:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013056:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233656.013057:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013058:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233656.013061:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233656.013064:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013066:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233656.013068:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233656.013071:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233656.013073:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013074:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233656.013076:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233656.013086:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233656.013090:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233656.013093:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233656.013094:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013097:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233656.013100:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233656.013104:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233656.013111:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233656.013118:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233656.013124:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245212160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233656.013132:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233656.013134:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233656.013137:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245212160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233656.013157:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013158:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233656.013160:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233656.013162:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233656.013162:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233656.013163:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233656.013164:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233656.013166:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233656.013168:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6125 00000100:00080000:0.0:1761233656.013169:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233656.013170:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013171:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233656.013172:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013173:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233656.013174:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233656.013175:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233656.013176:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233656.013177:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233656.013178:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233656.013179:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233656.013180:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013181:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233656.013182:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013184:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233656.013185:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233656.013187:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013189:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233656.013190:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233656.013195:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233656.013196:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013197:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233656.013200:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233656.013203:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013204:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233656.013206:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233656.013208:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233656.013209:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233656.013210:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013211:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233656.013212:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233656.013214:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233656.013216:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233656.013219:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233656.013222:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245212224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233656.013227:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233656.013228:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233656.013230:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245212224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233656.013234:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013235:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233656.013236:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233656.013238:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233656.013239:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233656.013241:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233656.013243:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233656.013245:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233656.013248:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6125 00000100:00080000:0.0:1761233656.013251:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233656.013255:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013258:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233656.013260:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013263:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233656.013265:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233656.013266:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233656.013269:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233656.013271:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233656.013273:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233656.013276:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233656.013278:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013280:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233656.013282:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013284:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233656.013287:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233656.013291:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233656.013294:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233656.013296:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233656.013298:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233656.013301:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013304:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233656.013306:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233656.013308:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013309:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233656.013312:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233656.013316:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233656.013318:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233656.013320:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013321:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233656.013322:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233656.013324:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233656.013330:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233656.013333:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233656.013337:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245212288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233656.013342:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233656.013343:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233656.013347:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245212288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233656.013356:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233656.013359:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233656.013364:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6135) 00000800:00000200:2.1:1761233656.013444:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233656.013447:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761233656.013683:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233656.013685:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761233656.013918:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233656.013919:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1761233656.909056:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233656.909062:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233656.909067:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233657.933065:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233657.933069:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233657.933071:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233658.957113:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233658.957117:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233658.957121:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233659.981029:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233659.981032:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233659.981035:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233661.005098:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233661.005102:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233661.005105:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233661.132937:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233661.132946:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233661.132948:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233661.132950:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233661.132952:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233661.132957:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233661.132962:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6130 00000100:00080000:0.0:1761233661.132965:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233661.132967:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.132968:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233661.132970:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.132972:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233661.132975:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233661.132976:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233661.132978:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233661.132981:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233661.132982:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233661.132984:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233661.132986:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.132987:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233661.132988:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.132989:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233661.132991:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233661.132994:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233661.132995:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233661.132997:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233661.133000:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233661.133001:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133003:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233661.133004:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233661.133012:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002ca01b78. 00000020:00000040:0.0:1761233661.133015:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233661.133018:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233661.133018:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133021:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233661.133022:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233661.133026:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233661.133031:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233661.133036:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233661.133041:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ca01b78 x1846784245212352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233661.133048:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233661.133049:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233661.133052:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002ca01b78] to pc [ptlrpcd_rcv+-1] req@000000002ca01b78 x1846784245212352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233661.133067:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133069:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233661.133070:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233661.133072:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233661.133073:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233661.133074:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233661.133075:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233661.133077:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233661.133078:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6130 00000100:00080000:0.0:1761233661.133079:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233661.133081:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133082:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233661.133083:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133084:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233661.133085:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233661.133085:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233661.133086:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233661.133087:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233661.133088:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233661.133089:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233661.133091:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133092:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233661.133092:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133093:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233661.133094:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233661.133095:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133097:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233661.133097:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233661.133101:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233661.133102:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133103:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233661.133105:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233661.133107:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133108:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233661.133110:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000095969832. 00000020:00000040:0.0:1761233661.133111:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233661.133113:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233661.133114:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133115:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233661.133116:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233661.133117:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233661.133120:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233661.133122:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233661.133125:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000095969832 x1846784245212416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233661.133129:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233661.133130:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233661.133132:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000095969832] to pc [ptlrpcd_rcv+-1] req@0000000095969832 x1846784245212416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233661.133139:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133141:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233661.133142:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233661.133146:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233661.133147:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233661.133148:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233661.133149:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233661.133152:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233661.133156:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6130 00000100:00080000:0.0:1761233661.133158:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233661.133162:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133164:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233661.133167:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133170:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233661.133174:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233661.133175:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233661.133178:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233661.133179:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233661.133181:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233661.133185:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233661.133190:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133192:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233661.133195:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133199:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233661.133201:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233661.133202:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233661.133203:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233661.133205:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233661.133207:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233661.133210:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133212:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233661.133213:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233661.133214:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133217:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233661.133221:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233661.133225:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233661.133226:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233661.133227:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133228:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233661.133230:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233661.133231:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233661.133235:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233661.133239:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233661.133244:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245212480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233661.133257:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233661.133259:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233661.133264:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245212480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233661.133274:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233661.133276:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233661.133280:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6140) 00000800:00000200:2.1:1761233661.133334:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233661.133336:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761233661.133601:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233661.133603:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761233661.133738:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233661.133740:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1761233662.029005:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233662.029009:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233662.029012:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233663.053060:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233663.053064:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233663.053068:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233664.077020:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233664.077023:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233664.077025:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233665.101051:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233665.101054:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233665.101057:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233666.125101:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233666.125105:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233666.125109:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233666.253027:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233666.253037:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233666.253039:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233666.253041:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233666.253043:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233666.253047:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233666.253056:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6135 00000100:00080000:0.0:1761233666.253059:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233666.253061:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253063:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233666.253065:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253067:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233666.253070:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233666.253072:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233666.253074:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233666.253077:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233666.253079:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233666.253081:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233666.253084:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253087:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233666.253090:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253093:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233666.253096:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233666.253101:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253103:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233666.253106:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233666.253108:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233666.253110:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253113:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233666.253115:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233666.253124:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233666.253127:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233666.253130:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233666.253131:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253134:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233666.253137:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233666.253140:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233666.253147:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233666.253153:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233666.253158:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245212544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233666.253165:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233666.253167:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233666.253179:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245212544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233666.253192:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253193:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233666.253195:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233666.253197:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233666.253198:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233666.253199:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233666.253199:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233666.253201:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233666.253202:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6135 00000100:00080000:0.0:1761233666.253204:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233666.253206:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253208:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233666.253209:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253211:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233666.253212:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233666.253213:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233666.253214:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233666.253216:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233666.253219:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233666.253220:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233666.253223:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253226:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233666.253229:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253231:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233666.253234:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233666.253238:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253242:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233666.253245:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233666.253249:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233666.253252:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253254:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233666.253258:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233666.253261:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253264:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233666.253267:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233666.253270:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233666.253273:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233666.253275:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253278:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233666.253279:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233666.253280:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233666.253284:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233666.253288:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233666.253292:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245212608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233666.253299:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233666.253301:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233666.253314:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245212608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233666.253337:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253339:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233666.253341:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233666.253345:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233666.253348:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233666.253350:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233666.253351:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233666.253354:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233666.253358:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6135 00000100:00080000:0.0:1761233666.253361:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233666.253365:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253367:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233666.253368:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253369:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233666.253370:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233666.253387:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233666.253389:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233666.253391:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233666.253394:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233666.253395:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233666.253400:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253404:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233666.253409:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253412:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233666.253415:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233666.253421:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233666.253423:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233666.253425:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233666.253428:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233666.253432:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253434:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233666.253435:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233666.253438:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253440:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233666.253443:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233666.253446:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233666.253450:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233666.253451:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253454:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233666.253456:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233666.253457:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761233666.253460:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000010:0.0:1761233666.253461:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233666.253464:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761233666.253465:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000100:00100000:0.0:1761233666.253471:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245212672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233666.253480:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233666.253481:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233666.253487:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245212672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233666.253497:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233666.253500:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233666.253503:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6145) 00000800:00000200:2.1:1761233666.253688:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233666.253689:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761233666.253818:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233666.253819:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233667.149086:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233667.149089:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233667.149091:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233668.173035:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233668.173038:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233668.173040:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233669.197018:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233669.197022:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233669.197025:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233670.221065:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233670.221067:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233670.221070:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233671.245037:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233671.245040:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233671.245042:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233671.373007:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233671.373019:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233671.373021:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233671.373024:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233671.373027:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233671.373032:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233671.373040:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6140 00000100:00080000:0.0:1761233671.373043:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233671.373045:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373049:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233671.373051:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373053:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233671.373056:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233671.373059:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233671.373061:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233671.373064:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233671.373066:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233671.373067:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233671.373069:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373070:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233671.373071:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373072:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233671.373074:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233671.373077:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373079:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233671.373082:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233671.373084:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233671.373086:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373088:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233671.373090:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233671.373100:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233671.373103:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233671.373107:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233671.373108:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373111:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233671.373114:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233671.373118:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233671.373125:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233671.373132:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233671.373139:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245212736/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233671.373151:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233671.373153:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233671.373157:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245212736/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233671.373174:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373176:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233671.373177:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233671.373179:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233671.373180:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233671.373181:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233671.373182:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233671.373184:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233671.373185:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6140 00000100:00080000:0.0:1761233671.373186:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233671.373188:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373189:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233671.373190:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373190:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233671.373192:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233671.373192:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233671.373193:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233671.373194:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233671.373195:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233671.373196:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233671.373198:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373199:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233671.373200:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373200:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233671.373201:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233671.373203:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373206:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233671.373207:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233671.373229:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233671.373231:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373233:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233671.373237:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233671.373240:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373242:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233671.373245:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233671.373247:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233671.373250:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233671.373251:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373252:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233671.373253:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233671.373255:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233671.373261:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233671.373266:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233671.373272:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245212800/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233671.373280:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233671.373282:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233671.373284:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245212800/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233671.373308:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373311:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233671.373312:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233671.373333:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233671.373335:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233671.373337:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233671.373339:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233671.373342:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233671.373347:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6140 00000100:00080000:0.0:1761233671.373351:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233671.373357:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373364:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233671.373368:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373373:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233671.373378:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233671.373380:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233671.373385:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233671.373393:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233671.373399:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233671.373402:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233671.373405:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373408:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233671.373413:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373417:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233671.373424:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233671.373432:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233671.373436:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233671.373443:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233671.373446:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233671.373449:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373450:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233671.373452:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233671.373453:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373454:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233671.373457:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233671.373460:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233671.373463:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233671.373465:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373469:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233671.373471:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233671.373472:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233671.373478:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233671.373482:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233671.373489:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245212864/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233671.373499:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233671.373500:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233671.373506:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245212864/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233671.373521:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233671.373531:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233671.373537:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6150) 00000800:00000200:2.1:1761233671.373612:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233671.373614:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761233671.373787:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233671.373788:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761233671.374042:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233671.374043:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1761233672.269058:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233672.269061:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233672.269064:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233673.293066:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233673.293069:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233673.293072:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233674.317047:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233674.317050:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233674.317052:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233675.341085:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233675.341088:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233675.341093:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233676.365117:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233676.365120:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233676.365123:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233676.492985:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233676.492995:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233676.492997:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233676.492999:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233676.493002:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233676.493007:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233676.493013:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6145 00000100:00080000:0.0:1761233676.493017:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233676.493021:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493024:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233676.493027:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493029:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233676.493033:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233676.493034:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233676.493037:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233676.493041:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233676.493044:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233676.493046:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233676.493051:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493054:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233676.493055:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493057:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233676.493059:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233676.493063:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493066:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233676.493068:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233676.493071:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233676.493073:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493075:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233676.493077:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233676.493085:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233676.493088:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233676.493092:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233676.493093:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493096:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233676.493099:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233676.493103:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233676.493110:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233676.493116:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233676.493122:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245212928/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233676.493131:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233676.493133:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233676.493137:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245212928/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233676.493155:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493157:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233676.493159:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233676.493162:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233676.493163:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233676.493164:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233676.493166:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233676.493168:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233676.493170:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6145 00000100:00080000:0.0:1761233676.493172:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233676.493174:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493176:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233676.493178:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493180:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233676.493182:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233676.493183:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233676.493185:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233676.493187:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233676.493189:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233676.493191:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233676.493194:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493196:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233676.493198:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493199:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233676.493203:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233676.493206:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493223:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233676.493227:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233676.493232:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233676.493237:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493240:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233676.493243:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233676.493248:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493252:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233676.493255:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233676.493259:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233676.493264:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233676.493267:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493270:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233676.493272:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233676.493275:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233676.493280:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233676.493286:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233676.493293:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245212992/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233676.493305:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233676.493307:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233676.493312:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245212992/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233676.493322:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493327:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233676.493331:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233676.493336:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233676.493338:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233676.493340:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233676.493342:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233676.493346:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233676.493351:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6145 00000100:00080000:0.0:1761233676.493356:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233676.493361:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493365:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233676.493369:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493372:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233676.493373:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233676.493374:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233676.493376:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233676.493378:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233676.493382:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233676.493385:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233676.493391:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493394:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233676.493398:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493403:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233676.493407:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233676.493413:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233676.493418:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233676.493422:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233676.493425:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233676.493430:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493433:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233676.493436:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233676.493441:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493446:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761233676.493453:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000010:0.0:1761233676.493454:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000800:00000200:2.1:1761233676.493456:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000020:00000040:0.0:1761233676.493459:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233676.493465:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233676.493468:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493472:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233676.493475:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233676.493478:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233676.493488:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233676.493493:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233676.493502:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245213056/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233676.493531:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233676.493535:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233676.493544:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245213056/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233676.493557:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233676.493562:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233676.493568:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6155) 00000800:00000200:2.1:1761233676.493724:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233676.493725:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761233676.493886:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233676.493888:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1761233677.389072:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233677.389075:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233677.389078:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233678.413139:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233678.413144:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233678.413149:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233679.437005:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233679.437008:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233679.437011:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233680.461092:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233680.461094:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233680.461098:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233681.485055:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233681.485060:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233681.485066:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233681.613012:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233681.613029:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233681.613031:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233681.613033:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233681.613036:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233681.613042:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233681.613048:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6150 00000100:00080000:0.0:1761233681.613051:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233681.613054:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613057:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233681.613059:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613061:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233681.613064:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233681.613066:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233681.613068:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233681.613071:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233681.613073:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233681.613075:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233681.613078:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613079:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233681.613081:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613082:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233681.613084:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233681.613088:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613090:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233681.613092:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233681.613095:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233681.613098:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613099:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233681.613102:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233681.613112:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233681.613116:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233681.613121:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233681.613122:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613125:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233681.613128:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233681.613151:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233681.613159:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233681.613165:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233681.613173:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245213120/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233681.613180:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233681.613182:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233681.613186:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245213120/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233681.613200:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613201:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233681.613203:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233681.613205:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233681.613206:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233681.613207:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233681.613208:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233681.613210:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233681.613212:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6150 00000100:00080000:0.0:1761233681.613213:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233681.613214:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613215:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233681.613216:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613217:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233681.613218:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233681.613219:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233681.613220:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233681.613221:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233681.613223:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233681.613224:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233681.613227:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613228:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233681.613230:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613231:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233681.613232:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233681.613234:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613235:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233681.613236:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233681.613241:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233681.613243:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613245:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233681.613248:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233681.613251:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613252:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233681.613254:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233681.613255:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233681.613257:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233681.613257:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613259:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233681.613260:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233681.613262:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233681.613265:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233681.613272:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233681.613278:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245213184/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233681.613285:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233681.613286:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233681.613292:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245213184/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233681.613361:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613374:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233681.613376:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233681.613380:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233681.613385:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233681.613388:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233681.613391:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233681.613398:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233681.613404:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6150 00000100:00080000:0.0:1761233681.613409:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233681.613420:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613422:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233681.613423:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613424:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233681.613426:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233681.613432:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233681.613441:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233681.613442:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233681.613446:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233681.613449:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233681.613454:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613458:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233681.613464:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613471:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233681.613483:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233681.613489:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233681.613494:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233681.613498:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233681.613500:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233681.613505:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613511:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233681.613519:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233681.613527:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613531:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233681.613535:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761233681.613537:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233681.613545:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233681.613548:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613552:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233681.613554:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233681.613556:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233681.613566:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233681.613574:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233681.613585:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245213248/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233681.613615:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233681.613620:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233681.613630:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245213248/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233681.613654:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233681.613663:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:2.1:1761233681.613664:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233681.613670:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000100:00000040:0.0:1761233681.613671:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6160) 00000800:00000200:2.1:1761233681.613963:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233681.613965:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761233681.614109:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233681.614111:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1761233682.509061:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233682.509064:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233682.509066:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233683.533056:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233683.533059:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233683.533062:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233684.557117:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233684.557121:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233684.557123:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233685.197025:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233685.197030:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233685.197031:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233685.197034:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233685.197039:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233685.197041:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233685.197044:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233685.197045:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233685.197048:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233685.581050:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233685.581053:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233685.581056:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233686.220989:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233686.220995:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233686.220996:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233686.221001:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233686.221006:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233686.221010:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233686.221012:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233686.221014:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233686.221016:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233686.605098:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233686.605102:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233686.605105:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233686.733419:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233686.733544:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233686.733553:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000400:00000200:0.0:1761233686.733607:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:0.0:1761233686.733628:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000800:00000200:2.1:1761233686.733658:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:0.0:1761233687.244926:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233687.244930:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233687.244931:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233687.244936:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233687.244947:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233687.244950:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233687.244952:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233687.244954:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233687.244956:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233687.629080:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233687.629082:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233687.629084:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233688.268958:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233688.268964:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233688.268966:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233688.268971:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233688.268975:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233688.268978:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233688.268981:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233688.268983:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233688.268986:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233688.653098:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233688.653103:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233688.653107:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233689.292943:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233689.292948:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233689.292950:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233689.292955:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233689.292960:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233689.292963:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233689.292965:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233689.292967:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233689.292970:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233689.677063:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233689.677066:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233689.677068:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233690.316988:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233690.317015:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233690.317016:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233690.317021:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233690.317027:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233690.317031:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233690.317033:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233690.317035:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233690.317038:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233690.701115:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233690.701118:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233690.701120:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233691.340939:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233691.340945:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233691.340946:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233691.340950:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233691.340955:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233691.340958:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233691.340960:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233691.340963:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233691.340965:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233691.725087:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233691.725091:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233691.725095:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233691.853580:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233691.853585:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233691.854004:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233691.854007:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233691.854239:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233691.854241:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1761233692.365007:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233692.365012:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233692.365014:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233692.365018:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233692.365024:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233692.365028:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233692.365030:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233692.365033:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233692.365035:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233692.749073:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233692.749078:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233692.749082:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233693.388967:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233693.388973:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233693.388975:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233693.388980:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233693.388985:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233693.388989:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233693.388991:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233693.388993:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233693.388995:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233693.773061:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233693.773064:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233693.773067:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233694.412997:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233694.413001:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233694.413003:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233694.413007:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233694.413011:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233694.413014:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233694.413017:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233694.413019:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233694.413021:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233694.797084:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233694.797088:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233694.797092:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233695.437011:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233695.437017:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233695.437019:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233695.437024:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233695.437030:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233695.437034:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233695.437036:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233695.437039:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233695.437041:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233695.821018:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233695.821021:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233695.821023:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233696.460959:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233696.460965:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233696.460967:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233696.460972:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233696.460977:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233696.460980:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233696.460983:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233696.460985:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233696.460987:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233696.845002:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233696.845005:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233696.845007:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233696.973408:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233696.973410:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761233696.973576:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233696.973577:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761233696.973764:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233696.973765:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000020:00000001:1.0:1761233697.869091:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233697.869094:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233697.869096:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233698.893054:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233698.893058:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233698.893061:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233699.917097:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233699.917100:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233699.917102:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233700.940984:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233700.940987:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233700.940992:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233701.965041:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233701.965044:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233701.965047:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233702.093376:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233702.093380:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1761233702.093681:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233702.093682:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1761233702.093834:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233702.093835:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000020:00000001:1.0:1761233702.989020:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233702.989024:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233702.989027:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233704.013068:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233704.013071:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233704.013074:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233705.037164:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233705.037168:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233705.037172:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233706.061019:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233706.061022:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233706.061025:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233707.085104:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233707.085107:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233707.085110:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233707.212955:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233707.212963:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233707.212965:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233707.212967:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233707.212969:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233707.212973:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233707.212978:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6176 00000100:00080000:1.0:1761233707.212981:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233707.212983:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.212985:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233707.212986:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.212988:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233707.212990:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233707.212992:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233707.212994:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233707.212997:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233707.212998:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233707.213000:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233707.213002:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213003:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233707.213004:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213004:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233707.213006:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233707.213008:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213009:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233707.213011:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233707.213013:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233707.213015:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213016:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233707.213018:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233707.213024:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761233707.213026:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233707.213030:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233707.213030:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213033:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233707.213034:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233707.213038:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233707.213043:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233707.213048:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233707.213053:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245214080/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233707.213061:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233707.213062:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233707.213065:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245214080/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233707.213079:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213080:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233707.213082:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233707.213084:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233707.213085:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233707.213086:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233707.213087:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233707.213088:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233707.213089:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6176 00000100:00080000:1.0:1761233707.213091:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233707.213092:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213093:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233707.213094:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213095:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233707.213096:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233707.213097:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233707.213097:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233707.213098:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233707.213099:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233707.213100:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233707.213102:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213103:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233707.213104:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213105:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233707.213106:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233707.213108:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213109:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233707.213110:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233707.213114:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233707.213116:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213118:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233707.213120:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233707.213123:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213125:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233707.213130:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761233707.213133:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233707.213137:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233707.213138:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213140:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233707.213142:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233707.213145:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233707.213150:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233707.213154:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233707.213158:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245214144/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233707.213169:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233707.213172:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233707.213180:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245214144/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233707.213236:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213243:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233707.213247:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233707.213253:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233707.213254:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233707.213256:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233707.213262:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233707.213269:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233707.213277:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6176 00000100:00080000:1.0:1761233707.213280:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233707.213285:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213288:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233707.213292:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213294:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233707.213297:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233707.213301:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233707.213303:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233707.213305:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233707.213308:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233707.213311:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233707.213314:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213316:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233707.213324:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213334:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233707.213347:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233707.213356:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233707.213364:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233707.213369:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233707.213373:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233707.213379:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213391:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233707.213394:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233707.213403:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213404:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233707.213408:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761233707.213410:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233707.213412:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233707.213414:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213416:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233707.213417:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233707.213420:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233707.213425:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233707.213428:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233707.213438:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245214208/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233707.213453:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233707.213456:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233707.213464:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245214208/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233707.213499:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233707.213522:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233707.213526:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6186) 00000800:00000200:2.1:1761233707.213545:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233707.213548:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761233707.213803:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233707.213805:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761233707.213977:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233707.213978:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1761233708.109100:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233708.109104:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233708.109108:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233709.133124:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233709.133128:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233709.133132:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233710.157091:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233710.157094:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233710.157098:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233711.181005:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233711.181007:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233711.181010:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233712.205052:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233712.205057:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233712.205060:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233712.333004:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233712.333021:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233712.333024:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233712.333027:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233712.333030:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233712.333038:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233712.333047:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6181 00000100:00080000:1.0:1761233712.333051:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233712.333055:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333058:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233712.333061:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333064:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233712.333068:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233712.333072:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233712.333075:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233712.333079:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233712.333081:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233712.333084:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233712.333087:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333089:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233712.333091:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333093:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233712.333096:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233712.333100:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333102:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233712.333106:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233712.333110:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233712.333113:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333116:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233712.333119:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233712.333130:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761233712.333134:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233712.333138:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233712.333139:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333142:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233712.333145:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233712.333150:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233712.333158:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233712.333165:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233712.333172:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245214272/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233712.333180:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233712.333182:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233712.333185:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245214272/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233712.333208:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333210:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233712.333212:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233712.333215:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233712.333215:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233712.333216:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233712.333217:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233712.333219:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233712.333220:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6181 00000100:00080000:1.0:1761233712.333222:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233712.333224:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333225:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233712.333226:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333227:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233712.333228:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233712.333229:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233712.333230:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233712.333231:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233712.333233:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233712.333234:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233712.333237:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333239:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233712.333241:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333242:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233712.333245:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233712.333247:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333250:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233712.333252:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233712.333260:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233712.333263:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333267:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233712.333272:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233712.333279:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333283:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233712.333291:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761233712.333296:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233712.333302:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233712.333305:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333308:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233712.333310:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233712.333313:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233712.333323:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233712.333329:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233712.333338:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245214336/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233712.333347:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233712.333350:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233712.333356:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245214336/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233712.333389:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333393:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233712.333396:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233712.333403:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233712.333405:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233712.333408:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233712.333410:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233712.333413:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233712.333415:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6181 00000100:00080000:1.0:1761233712.333417:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233712.333419:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333421:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233712.333425:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333428:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233712.333430:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233712.333431:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233712.333433:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233712.333435:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233712.333438:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233712.333441:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233712.333447:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333450:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233712.333454:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333456:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233712.333460:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233712.333462:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233712.333464:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233712.333466:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233712.333469:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233712.333473:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333476:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233712.333477:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233712.333480:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333482:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233712.333486:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000020:00000040:1.0:1761233712.333489:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233712.333491:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233712.333492:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333495:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233712.333496:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233712.333498:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233712.333502:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233712.333507:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761233712.333507:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233712.333511:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000100:00100000:1.0:1761233712.333514:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245214400/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233712.333524:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233712.333526:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233712.333530:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245214400/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233712.333539:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233712.333542:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233712.333545:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6191) 00000800:00000200:2.1:1761233712.333796:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233712.333798:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761233712.334043:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233712.334045:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1761233713.229054:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233713.229058:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233713.229061:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233714.253063:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233714.253066:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233714.253069:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233716.301041:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233716.301044:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233716.301046:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233717.453387:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233717.453390:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233717.453713:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233717.453714:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233717.453875:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233717.453876:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1761233718.349018:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233718.349021:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233718.349023:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233720.397038:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233720.397041:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233720.397045:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233722.445043:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233722.445046:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233722.445049:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233722.573002:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233722.573018:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233722.573021:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233722.573024:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233722.573027:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233722.573034:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233722.573044:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6191 00000100:00080000:1.0:1761233722.573048:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233722.573051:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573054:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233722.573058:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573061:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233722.573065:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233722.573068:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233722.573071:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233722.573075:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233722.573078:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233722.573081:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233722.573083:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573085:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233722.573087:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573090:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233722.573093:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233722.573097:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573099:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233722.573103:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233722.573106:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233722.573110:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573112:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233722.573114:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233722.573127:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761233722.573131:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233722.573136:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233722.573138:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573141:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233722.573145:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233722.573151:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233722.573161:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233722.573169:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233722.573177:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245214656/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233722.573188:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233722.573191:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233722.573196:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245214656/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233722.573225:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573227:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233722.573229:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233722.573233:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233722.573234:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233722.573236:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233722.573237:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233722.573240:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233722.573242:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6191 00000100:00080000:1.0:1761233722.573245:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233722.573247:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573250:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233722.573252:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573253:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233722.573256:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233722.573257:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233722.573258:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233722.573260:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233722.573262:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233722.573265:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233722.573268:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573271:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233722.573273:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573275:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233722.573277:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233722.573280:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573283:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233722.573285:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233722.573292:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233722.573295:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573297:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233722.573302:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233722.573306:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573308:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233722.573312:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761233722.573316:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233722.573321:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233722.573324:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573328:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233722.573330:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233722.573333:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233722.573339:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233722.573345:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233722.573355:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245214720/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233722.573389:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233722.573391:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233722.573396:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245214720/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233722.573411:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573416:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233722.573418:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233722.573427:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233722.573429:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233722.573433:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233722.573436:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233722.573443:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233722.573448:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6191 00000100:00080000:1.0:1761233722.573451:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233722.573456:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573461:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233722.573465:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573470:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233722.573475:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233722.573476:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233722.573479:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233722.573483:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233722.573504:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233722.573510:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233722.573520:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573527:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233722.573533:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573539:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233722.573548:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233722.573556:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233722.573564:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233722.573568:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233722.573572:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233722.573575:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573582:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233722.573586:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233722.573590:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573593:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233722.573600:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761233722.573606:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233722.573612:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233722.573613:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573618:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233722.573620:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233722.573623:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233722.573629:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233722.573636:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233722.573645:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245214784/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233722.573662:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233722.573664:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233722.573673:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245214784/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233722.573692:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233722.573696:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:2.1:1761233722.573697:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000040:1.0:1761233722.573702:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6201) 00000800:00000200:2.1:1761233722.573703:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761233722.574092:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233722.574094:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761233722.574302:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233722.574304:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1761233724.493036:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233724.493039:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233724.493041:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233726.541049:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233726.541051:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233726.541054:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233727.565055:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233727.565058:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233727.565060:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233727.693666:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233727.693671:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761233727.694141:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233727.694142:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761233727.694286:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233727.694287:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1761233728.589072:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233728.589077:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233728.589082:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233729.613074:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233729.613078:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233729.613081:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233730.637054:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233730.637058:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233730.637061:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233731.661038:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233731.661042:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233731.661044:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233732.685056:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233732.685058:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233732.685061:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233732.813401:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233732.813403:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761233732.813611:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233732.813612:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761233732.813740:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233732.813741:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233733.709107:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233733.709111:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233733.709115:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233734.733033:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233734.733036:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233734.733039:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233735.757030:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233735.757034:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233735.757037:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233736.781072:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233736.781076:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233736.781079:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233737.805016:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233737.805019:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233737.805023:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233737.932987:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233737.933000:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233737.933002:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233737.933005:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233737.933007:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233737.933013:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233737.933021:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6206 00000100:00080000:0.0:1761233737.933024:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233737.933026:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933029:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233737.933031:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933032:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233737.933036:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233737.933038:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233737.933041:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233737.933045:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233737.933046:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233737.933048:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233737.933050:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933051:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233737.933052:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933053:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233737.933055:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233737.933057:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933059:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233737.933061:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233737.933064:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233737.933066:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933068:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233737.933070:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233737.933081:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000006edc786f. 00000020:00000040:0.0:1761233737.933084:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233737.933088:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233737.933089:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933093:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233737.933095:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233737.933100:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233737.933110:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233737.933118:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233737.933127:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006edc786f x1846784245215232/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233737.933138:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233737.933140:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233737.933145:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000006edc786f] to pc [ptlrpcd_rcv+-1] req@000000006edc786f x1846784245215232/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233737.933168:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933171:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233737.933173:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233737.933177:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233737.933178:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233737.933180:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233737.933181:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233737.933183:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233737.933186:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6206 00000100:00080000:0.0:1761233737.933189:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233737.933192:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933194:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233737.933196:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933198:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233737.933201:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233737.933203:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233737.933204:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233737.933207:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233737.933208:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233737.933209:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233737.933212:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933213:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233737.933215:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933216:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233737.933219:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233737.933222:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933224:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233737.933227:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233737.933233:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233737.933238:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933240:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233737.933259:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233737.933262:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933263:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233737.933266:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ad5ae63. 00000020:00000040:0.0:1761233737.933268:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233737.933270:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233737.933271:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933272:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233737.933273:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233737.933274:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233737.933281:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233737.933286:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233737.933295:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ad5ae63 x1846784245215296/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233737.933327:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233737.933328:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233737.933336:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ad5ae63] to pc [ptlrpcd_rcv+-1] req@000000009ad5ae63 x1846784245215296/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233737.933348:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933357:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233737.933359:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233737.933367:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233737.933369:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233737.933374:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233737.933376:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233737.933387:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233737.933396:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6206 00000100:00080000:0.0:1761233737.933399:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233737.933403:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933407:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233737.933412:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933415:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233737.933418:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233737.933420:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233737.933421:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233737.933423:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233737.933425:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233737.933428:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233737.933434:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933438:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233737.933446:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933455:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233737.933462:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233737.933474:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233737.933480:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233737.933486:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233737.933490:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233737.933494:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933501:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233737.933505:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233737.933509:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933514:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233737.933517:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761233737.933521:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233737.933526:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233737.933527:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933529:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233737.933530:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233737.933532:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233737.933537:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233737.933551:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233737.933559:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245215360/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233737.933584:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233737.933589:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233737.933600:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245215360/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233737.933622:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233737.933628:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233737.933634:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6217) 00000800:00000200:2.1:1761233737.933648:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233737.933651:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1761233737.933916:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233737.933917:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000800:00000200:2.1:1761233737.934123:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233737.934124:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 11 secs later) 00000020:00000001:1.0:1761233738.829050:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233738.829053:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233738.829055:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233739.853072:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233739.853074:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233739.853077:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233740.877048:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233740.877052:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233740.877055:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233741.900987:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233741.900990:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233741.900994:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233742.925074:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233742.925079:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233742.925082:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233743.053467:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233743.053472:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1761233743.053817:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233743.053819:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:2.1:1761233743.054089:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233743.054091:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000020:00000001:1.0:1761233743.949049:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233743.949052:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233743.949055:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233744.973054:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233744.973057:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233744.973060:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233745.997067:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233745.997071:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233745.997074:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233747.021063:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233747.021066:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233747.021069:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233748.045079:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233748.045083:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233748.045125:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233748.173489:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233748.173494:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761233748.173743:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233748.173744:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:2.1:1761233748.173900:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233748.173901:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000020:00000001:1.0:1761233749.069085:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233749.069088:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233749.069091:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233750.093117:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233750.093121:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233750.093125:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233751.117097:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233751.117103:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233751.117108:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233751.756976:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233751.756983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233751.756985:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233751.756989:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233751.756995:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233751.756998:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233751.757000:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233751.757003:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233751.757005:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233752.141150:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233752.141155:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233752.141160:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233752.780986:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233752.780992:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233752.780993:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233752.780998:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233752.781004:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233752.781008:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233752.781047:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233752.781050:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233752.781053:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233753.165034:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233753.165038:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233753.165040:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233753.293057:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233753.293067:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233753.293068:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233753.293071:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233753.293073:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233753.293076:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233753.293082:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6222 00000100:00080000:0.0:1761233753.293084:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233753.293086:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293088:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233753.293089:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293091:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233753.293094:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233753.293095:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233753.293097:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233753.293100:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233753.293101:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233753.293103:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233753.293104:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293105:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233753.293106:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293107:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233753.293109:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233753.293111:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293113:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233753.293115:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233753.293118:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233753.293119:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293121:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233753.293122:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233753.293129:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233753.293131:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233753.293134:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233753.293135:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293137:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233753.293140:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233753.293143:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233753.293148:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233753.293153:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233753.293158:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245215808/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233753.293165:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233753.293167:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233753.293169:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245215808/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233753.293188:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293190:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233753.293191:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233753.293194:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233753.293194:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233753.293195:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233753.293196:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233753.293197:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233753.293199:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6222 00000100:00080000:0.0:1761233753.293200:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233753.293201:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293202:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233753.293203:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293204:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233753.293205:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233753.293206:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233753.293207:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233753.293208:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233753.293210:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233753.293211:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233753.293213:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293214:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233753.293215:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293217:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233753.293218:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233753.293220:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293221:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233753.293223:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233753.293227:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233753.293229:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293231:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233753.293234:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233753.293237:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293240:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233753.293242:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233753.293244:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233753.293248:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233753.293249:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293251:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233753.293252:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233753.293253:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233753.293258:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233753.293262:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233753.293267:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245215872/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233753.293273:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233753.293275:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233753.293280:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245215872/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233753.293289:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293292:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233753.293294:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233753.293299:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233753.293301:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233753.293302:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233753.293302:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233753.293304:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233753.293306:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6222 00000100:00080000:0.0:1761233753.293309:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233753.293312:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293314:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233753.293316:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293319:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233753.293321:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233753.293321:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233753.293322:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233753.293323:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233753.293326:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233753.293328:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233753.293330:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293332:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233753.293334:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293337:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233753.293340:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233753.293344:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233753.293346:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233753.293349:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233753.293352:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233753.293357:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293360:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233753.293363:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233753.293370:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293374:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233753.293379:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761233753.293383:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233753.293385:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233753.293386:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293391:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233753.293393:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233753.293395:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233753.293400:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233753.293405:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233753.293413:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245215936/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233753.293428:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233753.293430:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233753.293438:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245215936/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233753.293450:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233753.293453:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233753.293457:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6232) 00000800:00000200:2.1:1761233753.293481:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233753.293649:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233753.293818:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:0.0:1761233753.804979:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233753.804985:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233753.804986:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233753.804991:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233753.805009:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233753.805017:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233753.805019:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233753.805022:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233753.805025:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233754.189128:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233754.189133:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233754.189138:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233754.828967:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233754.828973:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233754.828975:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233754.828995:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233754.829000:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233754.829004:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233754.829006:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233754.829008:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233754.829010:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233755.213093:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233755.213098:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233755.213103:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233755.853022:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233755.853029:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233755.853031:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233755.853036:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233755.853041:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233755.853044:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233755.853047:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233755.853049:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233755.853051:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233756.237120:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233756.237124:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233756.237127:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233756.877033:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233756.877039:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233756.877042:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233756.877048:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233756.877055:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233756.877058:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233756.877060:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233756.877063:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233756.877066:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233757.261024:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233757.261029:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233757.261033:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233757.900929:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233757.900947:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233757.900949:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233757.900953:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233757.900957:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233757.900960:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233757.900962:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233757.900963:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233757.900966:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233758.285045:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233758.285048:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233758.285050:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233758.412954:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233758.412967:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233758.412970:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233758.412972:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233758.412974:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233758.412980:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233758.412986:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6227 00000100:00080000:0.0:1761233758.412991:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233758.412994:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.412996:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233758.412999:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413001:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233758.413005:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233758.413006:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233758.413009:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233758.413011:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233758.413013:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233758.413016:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233758.413019:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413021:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233758.413023:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413025:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233758.413027:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233758.413031:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413033:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233758.413036:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233758.413039:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233758.413041:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413044:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233758.413046:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233758.413052:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233758.413055:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233758.413059:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233758.413061:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413064:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233758.413066:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233758.413070:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233758.413077:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233758.413083:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233758.413090:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245216000/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233758.413100:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233758.413102:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233758.413106:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245216000/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233758.413134:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413136:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233758.413138:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233758.413141:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233758.413143:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233758.413144:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233758.413146:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233758.413148:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233758.413150:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6227 00000100:00080000:0.0:1761233758.413152:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233758.413155:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413157:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233758.413159:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413160:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233758.413162:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233758.413163:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233758.413165:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233758.413166:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233758.413168:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233758.413170:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233758.413172:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413174:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233758.413176:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413177:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233758.413179:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233758.413181:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413183:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233758.413185:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233758.413190:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233758.413192:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413194:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233758.413197:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233758.413200:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413201:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233758.413205:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233758.413207:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233758.413210:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233758.413210:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413212:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233758.413214:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233758.413215:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233758.413219:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233758.413222:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233758.413226:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245216064/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233758.413234:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233758.413235:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233758.413238:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245216064/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233758.413246:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413247:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233758.413248:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233758.413252:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233758.413253:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233758.413255:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233758.413256:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233758.413258:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233758.413261:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6227 00000100:00080000:0.0:1761233758.413263:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233758.413265:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413267:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233758.413269:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413271:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233758.413273:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233758.413274:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233758.413276:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233758.413277:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233758.413279:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233758.413281:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233758.413284:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413286:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233758.413287:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413289:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233758.413291:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233758.413293:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233758.413296:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233758.413297:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233758.413300:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233758.413301:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413303:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233758.413305:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233758.413307:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413309:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233758.413312:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233758.413314:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233758.413316:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233758.413317:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413319:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233758.413320:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233758.413322:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233758.413326:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233758.413328:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233758.413332:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245216128/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233758.413339:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233758.413340:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233758.413343:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245216128/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233758.413356:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233758.413358:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233758.413362:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6237) 00000800:00000200:2.1:1761233758.413621:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233758.413624:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233758.413952:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233758.413954:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233758.414192:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233758.414195:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1761233758.924973:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233758.924978:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233758.924979:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233758.924983:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233758.924987:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233758.924991:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233758.924993:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233758.924995:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233758.924996:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233759.309032:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233759.309036:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233759.309038:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233759.948983:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233759.948989:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233759.948991:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233759.948995:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233759.949000:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233759.949004:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233759.949006:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233759.949008:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233759.949010:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233760.973002:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233760.973008:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233760.973010:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233760.973014:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233760.973019:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233760.973022:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233760.973024:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233760.973026:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233760.973028:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233761.357068:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233761.357071:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233761.357073:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233761.996954:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233761.996958:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233761.996960:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233761.996963:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233761.996967:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233761.996970:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233761.996973:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233761.996975:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233761.996976:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233763.020950:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233763.020955:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233763.020956:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233763.020959:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233763.020963:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233763.020965:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233763.020967:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233763.020969:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233763.020971:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233763.405076:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233763.405081:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233763.405085:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233763.533587:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233763.533590:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761233763.533964:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233763.533965:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761233763.534143:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233763.534145:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1761233764.044995:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233764.045002:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233764.045004:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233764.045011:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233764.045018:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233764.045022:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233764.045025:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233764.045027:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233764.045030:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233765.069017:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233765.069026:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233765.069027:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233765.069034:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233765.069040:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233765.069045:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233765.069048:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233765.069050:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233765.069054:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233765.453091:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233765.453094:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233765.453096:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233766.092943:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233766.092949:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233766.092952:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233766.092957:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233766.092963:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233766.092967:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233766.092970:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233766.092973:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233766.092978:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233767.116933:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233767.116937:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233767.116939:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233767.116943:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233767.116948:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233767.116950:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233767.116952:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233767.116954:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233767.116955:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233767.501052:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233767.501056:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233767.501058:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233768.140944:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233768.140950:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233768.140952:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233768.140957:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233768.140962:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233768.140965:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233768.140967:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233768.140969:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233768.140971:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761233768.653451:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233768.653455:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233768.653756:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233768.653757:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233768.653945:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233768.653946:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000100:00000001:0.0:1761233769.164994:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233769.165000:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233769.165003:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233769.165007:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233769.165012:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233769.165015:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233769.165018:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233769.165020:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233769.165023:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233769.549114:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233769.549117:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233769.549120:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233770.188971:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233770.188977:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233770.188979:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233770.188984:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233770.188988:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233770.188992:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233770.188994:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233770.188996:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233770.188999:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233771.212910:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233771.212915:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233771.212917:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233771.212922:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233771.212928:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233771.212931:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233771.212933:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233771.212936:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233771.212938:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233772.236931:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233772.236937:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233772.236938:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233772.236943:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233772.236948:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233772.236952:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233772.236954:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233772.236956:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233772.236958:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233772.621045:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233772.621049:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233772.621053:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233773.261009:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233773.261018:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233773.261019:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233773.261024:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233773.261050:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233773.261054:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233773.261057:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233773.261059:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233773.261061:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761233773.772991:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233773.773003:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233773.773005:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233773.773008:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233773.773011:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233773.773017:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233773.773023:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6242 00000100:00080000:1.0:1761233773.773027:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233773.773029:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773031:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233773.773033:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773035:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233773.773038:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233773.773040:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233773.773043:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233773.773047:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233773.773049:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233773.773050:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233773.773053:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773054:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233773.773054:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773055:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233773.773057:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233773.773060:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773061:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233773.773064:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233773.773067:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233773.773069:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773070:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233773.773072:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233773.773082:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761233773.773086:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233773.773090:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233773.773091:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773094:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233773.773097:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233773.773101:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233773.773109:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233773.773115:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233773.773123:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245216576/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233773.773130:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233773.773132:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233773.773136:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245216576/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233773.773151:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773153:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233773.773155:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233773.773158:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233773.773159:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233773.773160:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233773.773160:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233773.773162:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233773.773163:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6242 00000100:00080000:1.0:1761233773.773164:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233773.773166:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773167:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233773.773168:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773169:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233773.773170:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233773.773171:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233773.773172:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233773.773173:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233773.773174:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233773.773175:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233773.773177:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773178:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233773.773179:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773180:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233773.773181:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233773.773182:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773183:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233773.773184:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233773.773189:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233773.773190:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773191:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233773.773195:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233773.773198:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773199:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233773.773201:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761233773.773202:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233773.773204:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233773.773205:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773206:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233773.773208:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233773.773209:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233773.773214:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233773.773216:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233773.773219:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245216640/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233773.773224:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233773.773225:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233773.773227:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245216640/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233773.773232:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773233:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233773.773233:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233773.773235:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233773.773236:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233773.773236:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233773.773237:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233773.773240:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233773.773246:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6242 00000100:00080000:1.0:1761233773.773248:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233773.773250:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773253:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233773.773254:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773258:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233773.773260:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233773.773262:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233773.773264:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233773.773266:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233773.773270:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233773.773276:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233773.773285:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773291:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233773.773296:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773302:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233773.773308:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233773.773319:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233773.773324:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233773.773329:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233773.773338:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233773.773343:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773348:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233773.773354:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233773.773361:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773364:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233773.773371:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000020:00000040:1.0:1761233773.773377:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233773.773384:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233773.773385:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773392:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233773.773394:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233773.773397:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233773.773404:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233773.773410:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233773.773420:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245216704/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233773.773433:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233773.773435:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233773.773439:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245216704/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233773.773464:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233773.773472:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233773.773478:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6252) 00000800:00000200:2.1:1761233773.773518:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233773.773520:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761233773.773710:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233773.773711:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761233773.773991:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233773.773993:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000100:00000001:0.0:1761233774.285005:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233774.285013:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233774.285015:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233774.285021:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233774.285027:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233774.285032:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233774.285035:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233774.285037:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233774.285040:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233774.669079:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233774.669082:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233774.669085:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233775.308976:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233775.308983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233775.308985:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233775.308990:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233775.308994:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233775.308997:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233775.308999:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233775.309000:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233775.309002:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233776.332985:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233776.332991:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233776.332993:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233776.332997:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233776.333002:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233776.333005:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233776.333008:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233776.333011:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233776.333013:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233776.717010:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233776.717013:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233776.717016:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233777.356965:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233777.356971:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233777.356973:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233777.356977:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233777.356982:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233777.356985:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233777.356988:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233777.356990:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233777.356992:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233778.765062:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233778.765066:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233778.765069:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233778.892949:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233778.892958:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233778.892959:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233778.892962:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233778.892963:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233778.892967:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233778.892972:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6247 00000100:00080000:1.0:1761233778.892975:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233778.892977:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.892979:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233778.892981:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.892983:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233778.892986:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233778.892988:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233778.892990:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233778.892992:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233778.892994:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233778.892996:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233778.892999:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893000:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233778.893002:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893004:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233778.893005:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233778.893008:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893009:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233778.893011:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233778.893013:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233778.893015:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893017:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233778.893019:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233778.893026:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761233778.893028:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233778.893032:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233778.893033:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893035:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233778.893038:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233778.893041:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233778.893047:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233778.893052:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233778.893057:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245216768/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233778.893063:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233778.893065:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233778.893067:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245216768/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233778.893079:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893080:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233778.893081:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233778.893083:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233778.893084:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233778.893085:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233778.893086:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233778.893088:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233778.893089:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6247 00000100:00080000:1.0:1761233778.893090:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233778.893092:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893093:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233778.893094:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893094:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233778.893096:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233778.893096:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233778.893097:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233778.893098:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233778.893099:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233778.893100:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233778.893102:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893104:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233778.893105:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893106:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233778.893108:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233778.893109:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893110:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233778.893112:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233778.893116:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233778.893118:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893120:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233778.893123:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233778.893126:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893128:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233778.893132:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761233778.893135:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233778.893138:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233778.893139:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893142:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233778.893143:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233778.893145:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233778.893149:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233778.893153:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233778.893158:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245216832/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233778.893176:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233778.893178:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233778.893182:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245216832/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233778.893192:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893194:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233778.893195:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233778.893199:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233778.893200:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233778.893202:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233778.893203:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233778.893205:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233778.893207:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6247 00000100:00080000:1.0:1761233778.893210:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233778.893214:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893216:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233778.893220:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893224:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233778.893227:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233778.893228:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233778.893229:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233778.893231:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233778.893234:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233778.893239:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233778.893243:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893246:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233778.893249:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893251:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233778.893252:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233778.893255:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233778.893258:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233778.893260:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233778.893262:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233778.893263:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893264:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233778.893265:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233778.893266:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893268:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233778.893271:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761233778.893275:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233778.893277:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233778.893278:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893279:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233778.893281:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233778.893283:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233778.893286:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233778.893290:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233778.893296:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245216896/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233778.893303:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233778.893305:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233778.893310:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245216896/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233778.893321:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233778.893323:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233778.893327:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6258) 00000800:00000200:2.1:1761233778.893351:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233778.893353:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761233778.893639:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233778.893641:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:2.1:1761233778.893827:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233778.893830:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1761233780.813051:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233780.813055:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233780.813057:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233782.861033:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233782.861037:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233782.861040:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233784.013572:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233784.013575:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233784.013776:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233784.013777:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761233784.014013:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233784.014014:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1761233784.909027:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233784.909029:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233784.909031:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233786.957101:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233786.957105:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233786.957109:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233789.005109:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233789.005112:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233789.005115:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233789.132955:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233789.132967:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233789.132969:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233789.132972:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233789.132974:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233789.132980:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233789.132986:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6258 00000100:00080000:1.0:1761233789.132989:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233789.132993:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.132995:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233789.132997:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.132999:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233789.133002:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233789.133004:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233789.133006:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233789.133008:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233789.133011:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233789.133013:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233789.133016:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133018:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233789.133020:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133021:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233789.133024:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233789.133027:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133030:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233789.133033:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233789.133036:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233789.133038:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133040:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233789.133042:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233789.133051:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f8a85602. 00000020:00000040:1.0:1761233789.133054:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233789.133059:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233789.133060:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133062:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233789.133065:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233789.133069:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233789.133076:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233789.133083:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233789.133090:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f8a85602 x1846784245217152/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233789.133099:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233789.133101:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233789.133105:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f8a85602] to pc [ptlrpcd_rcv+-1] req@00000000f8a85602 x1846784245217152/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233789.133128:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133130:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233789.133132:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233789.133140:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233789.133141:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233789.133143:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233789.133145:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233789.133147:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233789.133149:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6258 00000100:00080000:1.0:1761233789.133151:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233789.133154:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133157:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233789.133160:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133162:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233789.133164:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233789.133165:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233789.133167:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233789.133169:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233789.133172:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233789.133176:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233789.133182:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133186:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233789.133191:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133197:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233789.133202:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233789.133207:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133213:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233789.133218:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233789.133226:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233789.133233:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133236:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233789.133240:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233789.133245:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133249:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233789.133254:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761233789.133259:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233789.133262:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233789.133265:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133268:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233789.133269:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233789.133271:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233789.133278:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233789.133285:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233789.133293:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245217216/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233789.133307:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233789.133310:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233789.133316:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245217216/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233789.133331:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133334:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233789.133337:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233789.133340:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233789.133341:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233789.133342:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233789.133344:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233789.133347:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233789.133354:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6258 00000100:00080000:1.0:1761233789.133360:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233789.133366:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133372:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233789.133377:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133381:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233789.133386:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233789.133388:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233789.133391:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233789.133394:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233789.133398:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233789.133404:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233789.133408:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133413:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233789.133418:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000800:00000200:2.1:1761233789.133420:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:1.0:1761233789.133422:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000800:00000200:2.1:1761233789.133424:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000020:00000040:1.0:1761233789.133426:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233789.133433:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233789.133452:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233789.133455:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233789.133458:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233789.133460:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133462:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233789.133464:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233789.133468:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133472:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233789.133478:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761233789.133484:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233789.133490:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233789.133493:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133498:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233789.133501:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233789.133504:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233789.133510:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233789.133517:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233789.133525:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245217280/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233789.133562:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233789.133566:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233789.133575:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245217280/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233789.133609:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233789.133613:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233789.133618:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6268) 00000800:00000200:2.1:1761233789.133770:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233789.133771:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761233789.133938:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233789.133939:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1761233791.053080:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233791.053083:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233791.053086:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233793.101032:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233793.101034:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233793.101037:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233794.253018:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233794.253035:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233794.253037:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233794.253040:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233794.253043:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233794.253048:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233794.253054:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6263 00000100:00080000:0.0:1761233794.253057:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233794.253061:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253064:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233794.253066:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253069:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233794.253073:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233794.253075:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233794.253077:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233794.253081:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233794.253083:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233794.253085:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233794.253089:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253091:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233794.253094:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253095:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233794.253097:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233794.253100:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253101:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233794.253104:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233794.253106:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233794.253108:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253110:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233794.253112:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233794.253120:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233794.253123:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233794.253127:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233794.253128:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253130:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233794.253133:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233794.253136:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233794.253145:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233794.253151:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233794.253157:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245217344/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233794.253166:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233794.253167:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233794.253171:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245217344/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233794.253186:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253189:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233794.253191:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233794.253196:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233794.253197:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233794.253198:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233794.253199:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233794.253202:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233794.253205:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6263 00000100:00080000:0.0:1761233794.253207:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233794.253210:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253213:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233794.253215:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253217:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233794.253219:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233794.253221:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233794.253222:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233794.253224:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233794.253227:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233794.253229:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233794.253233:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253235:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233794.253237:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253239:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233794.253241:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233794.253244:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253247:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233794.253249:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233794.253255:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233794.253257:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253260:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233794.253263:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233794.253267:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253269:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233794.253273:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233794.253276:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233794.253279:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233794.253291:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253294:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233794.253295:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233794.253297:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233794.253302:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233794.253306:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233794.253309:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245217408/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233794.253317:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233794.253318:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233794.253321:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245217408/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233794.253329:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253331:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233794.253333:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233794.253337:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233794.253339:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233794.253340:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233794.253342:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233794.253345:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233794.253347:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6263 00000100:00080000:0.0:1761233794.253350:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233794.253353:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253356:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233794.253390:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253393:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233794.253395:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233794.253397:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233794.253399:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233794.253401:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233794.253402:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233794.253404:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233794.253408:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253410:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233794.253412:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253414:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233794.253416:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233794.253419:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233794.253421:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233794.253424:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233794.253426:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233794.253429:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253431:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233794.253433:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233794.253436:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253438:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233794.253442:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233794.253444:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233794.253447:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233794.253448:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253451:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233794.253452:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233794.253454:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233794.253458:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233794.253461:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233794.253465:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245217472/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233794.253473:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233794.253475:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233794.253478:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245217472/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:2.1:1761233794.253483:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233794.253486:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1761233794.253487:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233794.253489:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233794.253494:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6273) 00000800:00000200:2.1:1761233794.253773:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233794.253775:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761233794.253985:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233794.253987:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1761233795.149043:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233795.149046:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233795.149049:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233797.197077:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233797.197080:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233797.197083:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233798.221019:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233798.221024:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233798.221027:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233799.245039:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233799.245042:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233799.245044:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233799.373380:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233799.373384:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233799.373691:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233799.373693:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233799.373927:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233799.373929:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233800.269044:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233800.269048:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233800.269051:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233801.293136:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233801.293141:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233801.293144:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233802.317122:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233802.317126:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233802.317133:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233803.341052:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233803.341056:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233803.341059:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233804.365007:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233804.365010:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233804.365013:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233804.493354:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233804.493359:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761233804.493647:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233804.493649:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761233804.493787:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233804.493788:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1761233805.389026:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233805.389030:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233805.389033:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233806.413030:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233806.413035:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233806.413038:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233807.437050:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233807.437052:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233807.437055:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233808.461113:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233808.461116:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233808.461118:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233809.485060:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233809.485063:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233809.485065:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233809.613044:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233809.613057:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233809.613059:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233809.613062:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233809.613064:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233809.613069:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233809.613075:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6278 00000100:00080000:1.0:1761233809.613079:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233809.613081:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613084:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233809.613085:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613088:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233809.613091:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233809.613093:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233809.613095:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233809.613098:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233809.613100:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233809.613102:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233809.613105:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613106:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233809.613109:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613110:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233809.613113:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233809.613131:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613134:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233809.613136:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233809.613139:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233809.613140:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613142:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233809.613144:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233809.613152:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761233809.613154:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233809.613157:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233809.613158:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613160:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233809.613163:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233809.613165:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233809.613171:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233809.613176:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233809.613181:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245217920/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233809.613188:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233809.613190:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233809.613192:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245217920/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233809.613210:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613211:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233809.613213:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233809.613215:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233809.613215:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233809.613216:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233809.613217:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233809.613219:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233809.613221:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6278 00000100:00080000:1.0:1761233809.613222:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233809.613224:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613225:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233809.613226:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613227:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233809.613228:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233809.613229:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233809.613229:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233809.613231:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233809.613232:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233809.613233:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233809.613234:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613235:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233809.613236:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613237:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233809.613239:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233809.613240:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613242:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233809.613243:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233809.613247:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233809.613249:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613250:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233809.613252:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233809.613255:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613256:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233809.613258:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000020:00000040:1.0:1761233809.613260:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233809.613262:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233809.613264:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613265:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233809.613266:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233809.613268:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233809.613271:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233809.613274:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233809.613279:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245217984/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233809.613293:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233809.613294:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233809.613298:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245217984/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233809.613305:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613308:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233809.613309:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233809.613313:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233809.613314:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233809.613316:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233809.613316:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233809.613318:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233809.613319:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6278 00000100:00080000:1.0:1761233809.613322:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233809.613326:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613329:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233809.613331:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613334:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233809.613337:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233809.613337:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233809.613339:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233809.613341:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233809.613342:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233809.613345:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233809.613348:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613350:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233809.613351:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613352:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233809.613357:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233809.613362:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233809.613363:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233809.613364:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233809.613366:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233809.613369:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613371:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233809.613372:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233809.613377:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613379:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233809.613383:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761233809.613387:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233809.613392:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233809.613395:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613398:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233809.613400:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233809.613402:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233809.613407:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233809.613412:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233809.613420:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245218048/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233809.613433:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233809.613435:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233809.613441:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245218048/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233809.613449:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233809.613451:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233809.613456:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6288) 00000800:00000200:2.1:1761233809.613515:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233809.613518:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761233809.613832:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233809.613834:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761233809.614090:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233809.614092:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1761233810.509080:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233810.509083:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233810.509086:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233811.533130:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233811.533132:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233811.533135:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233812.557066:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233812.557068:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233812.557093:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233813.581042:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233813.581045:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233813.581047:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233814.605095:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233814.605098:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233814.605100:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233814.732932:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233814.732941:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233814.732942:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233814.732944:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233814.732946:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233814.732950:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233814.732956:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6283 00000100:00080000:1.0:1761233814.732959:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233814.732961:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.732962:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233814.732964:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.732965:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233814.732968:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233814.732969:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233814.732971:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233814.732973:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233814.732975:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233814.732976:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233814.732978:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.732979:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233814.732981:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.732982:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233814.732983:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233814.732986:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233814.732987:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233814.732989:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233814.732992:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233814.732993:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.732995:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233814.732996:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233814.733003:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761233814.733005:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233814.733008:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233814.733009:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733011:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233814.733013:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233814.733016:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233814.733021:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233814.733025:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233814.733030:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245218112/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233814.733066:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233814.733068:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233814.733071:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245218112/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233814.733090:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733091:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233814.733093:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233814.733095:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233814.733096:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233814.733097:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233814.733098:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233814.733099:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233814.733101:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6283 00000100:00080000:1.0:1761233814.733102:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233814.733104:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733106:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233814.733107:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733107:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233814.733109:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233814.733109:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233814.733110:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233814.733111:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233814.733113:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233814.733114:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233814.733115:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733116:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233814.733117:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733118:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233814.733119:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233814.733120:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733122:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233814.733123:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233814.733128:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233814.733129:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733131:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233814.733133:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233814.733135:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733137:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233814.733139:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761233814.733141:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233814.733144:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233814.733145:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733147:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233814.733150:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233814.733152:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233814.733157:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233814.733161:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233814.733168:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245218176/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233814.733178:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233814.733180:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233814.733183:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245218176/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233814.733197:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733199:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233814.733202:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233814.733206:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233814.733207:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233814.733208:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233814.733209:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233814.733212:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233814.733216:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6283 00000100:00080000:1.0:1761233814.733219:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233814.733221:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733224:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233814.733225:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733226:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233814.733228:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233814.733230:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233814.733231:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233814.733232:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233814.733234:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233814.733240:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233814.733244:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733247:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233814.733251:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733255:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233814.733258:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233814.733262:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233814.733265:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233814.733268:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233814.733271:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233814.733276:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733279:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233814.733281:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233814.733283:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733286:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233814.733291:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761233814.733292:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233814.733294:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233814.733294:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733295:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233814.733296:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233814.733297:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233814.733300:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233814.733302:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233814.733307:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245218240/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233814.733315:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233814.733317:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233814.733321:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245218240/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233814.733338:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233814.733340:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233814.733343:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6293) 00000800:00000200:2.1:1761233814.733378:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233814.733381:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761233814.733635:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233814.733636:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761233814.733779:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233814.733780:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1761233815.629014:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233815.629018:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233815.629033:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233816.653069:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233816.653072:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233816.653075:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233817.677074:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233817.677077:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233817.677079:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233818.701071:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233818.701074:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233818.701077:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233819.725088:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233819.725091:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233819.725094:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233819.853506:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:1.0:1761233819.853606:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000400:00000200:1.0:1761233819.853666:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000800:00000200:2.1:1761233819.853679:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:1.0:1761233819.853687:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000800:00000200:2.1:1761233819.853812:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000020:00000001:1.0:1761233820.749107:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233820.749110:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233820.749113:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233821.773113:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233821.773116:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233821.773119:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233822.797131:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233822.797135:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233822.797139:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1761233822.925264:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:1.0:1761233822.925309:0:88985:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1761233822.925319:0:88985:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:1.0:1761233822.925330:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:1.0:1761233822.925334:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007ceb0618 (tot 1676778). 00000400:00000200:1.0:1761233822.925341:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1761233822.925346:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1761233822.925352:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bf8b045f 00000400:00000010:1.0:1761233822.925354:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bf8b045f. 00000100:00000001:1.0:1761233822.925361:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233822.925369:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000266eaa93 x1846784245218304/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233844 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233822.925396:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233822.925400:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000266eaa93 x1846784245218304/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233844 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233822.925408:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233822.925410:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:1.0:1761233822.925430:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:1.0:1761233822.925432:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676530). 00000400:00000200:1.0:1761233822.925435:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1761233822.925439:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1761233822.925441:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000f98e6c57 00000400:00000010:1.0:1761233822.925443:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000f98e6c57. 00000100:00000001:1.0:1761233822.925446:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233822.925450:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000db341c88 x1846784245218368/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233844 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233822.925459:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233822.925462:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000db341c88 x1846784245218368/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233844 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233822.925468:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233822.925472:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:1.0:1761233822.925479:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:1.0:1761233822.925483:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:1.0:1761233822.925488:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1761233822.925497:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1761233822.925504:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000002b6caf4f 00000400:00000010:1.0:1761233822.925506:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000002b6caf4f. 00000100:00000001:1.0:1761233822.925508:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761233822.925512:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000004b242e46 x1846784245218432/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233844 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233822.925523:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761233822.925527:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004b242e46 x1846784245218432/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233844 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233822.925538:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233822.925540:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000020:00000001:1.0:1761233823.821006:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233823.821009:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233823.821011:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233824.845089:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233824.845091:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233824.845094:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233824.973017:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233824.973025:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233824.973027:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233824.973029:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233824.973031:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233824.973035:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233824.973039:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6293 00000100:00080000:1.0:1761233824.973042:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233824.973044:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973046:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233824.973047:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973049:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233824.973051:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233824.973053:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233824.973055:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233824.973057:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233824.973058:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233824.973059:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233824.973061:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973062:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233824.973063:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973064:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233824.973067:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233824.973069:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973071:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233824.973073:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233824.973075:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233824.973077:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973078:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233824.973079:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233824.973083:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761233824.973085:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233824.973089:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233824.973090:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973092:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233824.973094:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233824.973096:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233824.973102:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233824.973107:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233824.973112:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245218496/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233824.973119:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233824.973121:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233824.973123:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245218496/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233824.973136:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973138:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233824.973139:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233824.973141:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233824.973141:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233824.973142:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233824.973143:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233824.973145:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233824.973146:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6293 00000100:00080000:1.0:1761233824.973148:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233824.973149:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973151:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233824.973152:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973153:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233824.973154:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233824.973155:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233824.973155:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233824.973156:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233824.973157:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233824.973158:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233824.973160:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973161:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233824.973162:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973163:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233824.973164:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233824.973165:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973167:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233824.973168:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233824.973172:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233824.973173:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973174:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233824.973177:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233824.973179:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973180:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233824.973182:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000020:00000040:1.0:1761233824.973183:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233824.973185:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233824.973185:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973186:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233824.973187:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233824.973188:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233824.973192:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233824.973197:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233824.973204:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245218560/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233824.973219:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233824.973221:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233824.973227:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245218560/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233824.973238:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973241:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233824.973244:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233824.973251:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233824.973253:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233824.973256:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233824.973258:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233824.973262:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233824.973267:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6293 00000100:00080000:1.0:1761233824.973271:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233824.973274:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973279:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233824.973282:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973285:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233824.973287:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233824.973289:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233824.973292:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233824.973294:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233824.973297:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233824.973301:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233824.973303:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973306:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233824.973310:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973314:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233824.973318:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233824.973330:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233824.973335:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233824.973340:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233824.973344:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233824.973349:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973353:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233824.973356:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233824.973359:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973362:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233824.973367:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761233824.973371:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233824.973376:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233824.973377:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973378:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233824.973380:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233824.973382:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233824.973388:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233824.973391:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233824.973402:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245218624/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233824.973416:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233824.973418:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233824.973425:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245218624/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233824.973443:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233824.973446:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233824.973451:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6304) 00000800:00000200:2.1:1761233824.973474:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233824.973477:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233824.973784:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233824.973786:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233824.974057:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233824.974059:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1761233825.869056:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233825.869059:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233825.869062:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233826.893029:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233826.893032:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233826.893035:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233827.917032:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233827.917035:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233827.917037:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233828.941074:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233828.941078:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233828.941081:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233829.965111:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233829.965114:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233829.965118:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233830.093608:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233830.093613:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761233830.094032:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233830.094034:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761233830.094271:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233830.094273:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1761233830.989084:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233830.989089:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233830.989094:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233832.013114:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233832.013118:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233832.013120:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233833.037085:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233833.037091:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233833.037097:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233834.061113:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233834.061117:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233834.061120:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233835.085026:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233835.085030:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233835.085033:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233835.213482:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233835.213486:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233835.213902:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233835.213905:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233835.214159:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233835.214161:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1761233836.109071:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233836.109075:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233836.109079:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233837.133074:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233837.133077:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233837.133080:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233838.157070:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233838.157074:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233838.157078:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233839.181133:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233839.181138:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233839.181144:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233840.205100:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233840.205104:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233840.205108:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233840.333348:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233840.333351:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761233840.333557:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233840.333558:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761233840.333759:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233840.333761:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1761233842.253007:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233842.253010:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233842.253012:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233844.301020:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233844.301024:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233844.301027:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233845.325051:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233845.325055:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233845.325059:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233845.453036:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233845.453048:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233845.453050:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233845.453053:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233845.453056:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233845.453063:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233845.453070:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6314 00000100:00080000:1.0:1761233845.453073:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233845.453075:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453078:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233845.453079:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453081:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233845.453084:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233845.453086:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233845.453088:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233845.453091:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233845.453093:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233845.453095:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233845.453097:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453098:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233845.453099:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453101:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233845.453104:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233845.453107:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453109:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233845.453112:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233845.453115:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233845.453118:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453120:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233845.453121:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233845.453133:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761233845.453137:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233845.453141:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233845.453142:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453151:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233845.453153:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233845.453158:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233845.453168:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233845.453175:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233845.453182:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245219264/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233845.453190:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233845.453192:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233845.453196:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245219264/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233845.453216:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453218:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233845.453219:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233845.453222:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233845.453222:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233845.453223:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233845.453224:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233845.453226:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233845.453228:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6314 00000100:00080000:1.0:1761233845.453230:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233845.453232:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453233:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233845.453234:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453236:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233845.453237:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233845.453238:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233845.453239:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233845.453240:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233845.453241:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233845.453242:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233845.453244:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453245:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233845.453246:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453247:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233845.453248:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233845.453249:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453250:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233845.453251:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233845.453257:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233845.453258:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453259:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233845.453262:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233845.453265:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453266:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233845.453268:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761233845.453269:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233845.453271:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233845.453272:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453273:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233845.453274:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233845.453275:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233845.453277:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233845.453280:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233845.453282:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245219328/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233845.453287:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233845.453289:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233845.453290:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245219328/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233845.453297:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453298:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233845.453298:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233845.453301:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233845.453301:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233845.453302:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233845.453303:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233845.453305:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233845.453308:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6314 00000100:00080000:1.0:1761233845.453310:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233845.453314:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453316:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233845.453317:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453318:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233845.453320:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233845.453320:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233845.453321:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233845.453322:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233845.453324:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233845.453325:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233845.453326:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453327:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233845.453329:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453333:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233845.453339:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233845.453343:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233845.453347:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233845.453349:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233845.453352:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233845.453354:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453356:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233845.453357:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233845.453361:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453363:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233845.453368:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761233845.453370:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233845.453374:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233845.453375:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453379:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233845.453380:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233845.453382:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233845.453389:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233845.453396:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233845.453409:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245219392/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233845.453421:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233845.453423:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233845.453431:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245219392/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233845.453440:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233845.453445:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233845.453448:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6324) 00000800:00000200:2.1:1761233845.453587:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233845.453589:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761233845.453764:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233845.453765:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761233845.454014:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233845.454016:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1761233846.349054:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233846.349059:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233846.349065:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233847.373069:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233847.373071:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233847.373074:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233848.397049:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233848.397053:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233848.397057:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233849.421059:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233849.421062:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233849.421065:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233850.445025:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233850.445029:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233850.445032:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233850.572956:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233850.572965:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233850.572966:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233850.572969:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233850.572971:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233850.572975:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233850.572980:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6319 00000100:00080000:1.0:1761233850.572983:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233850.572985:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.572987:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233850.572988:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.572990:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233850.572993:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233850.572994:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233850.572996:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233850.572998:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233850.573000:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233850.573001:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233850.573003:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573004:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233850.573005:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573007:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233850.573008:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233850.573011:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573012:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233850.573014:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233850.573017:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233850.573019:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573020:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233850.573022:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233850.573030:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761233850.573033:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233850.573036:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233850.573037:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573039:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233850.573042:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233850.573045:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233850.573051:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233850.573056:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233850.573061:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245219456/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233850.573068:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233850.573070:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233850.573072:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245219456/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233850.573089:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573090:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233850.573092:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233850.573095:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233850.573096:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233850.573098:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233850.573099:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233850.573101:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233850.573103:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6319 00000100:00080000:1.0:1761233850.573105:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233850.573107:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573109:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233850.573111:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573112:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233850.573115:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233850.573116:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233850.573117:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233850.573119:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233850.573121:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233850.573123:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233850.573125:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573127:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233850.573128:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573130:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233850.573131:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233850.573132:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573134:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233850.573135:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233850.573140:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233850.573141:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573142:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233850.573145:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233850.573147:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573149:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233850.573153:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761233850.573155:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233850.573157:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233850.573158:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573159:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233850.573160:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233850.573162:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233850.573166:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233850.573170:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233850.573176:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245219520/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233850.573199:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233850.573201:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233850.573205:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245219520/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233850.573214:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573215:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233850.573225:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233850.573231:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233850.573233:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233850.573235:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233850.573236:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233850.573239:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233850.573243:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6319 00000100:00080000:1.0:1761233850.573245:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233850.573247:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573248:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233850.573250:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573254:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233850.573257:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233850.573258:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233850.573259:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233850.573260:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233850.573262:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233850.573265:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233850.573266:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573268:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233850.573271:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573273:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233850.573274:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233850.573276:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233850.573280:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233850.573283:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233850.573297:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233850.573299:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573301:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233850.573303:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233850.573306:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573309:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233850.573313:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761233850.573316:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233850.573320:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233850.573321:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573323:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233850.573325:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233850.573327:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233850.573332:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233850.573336:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233850.573340:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245219584/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233850.573345:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233850.573346:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233850.573349:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245219584/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233850.573359:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233850.573360:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233850.573363:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6329) 00000800:00000200:2.1:1761233850.573403:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233850.573405:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761233850.573628:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233850.573629:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761233850.573759:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233850.573769:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000020:00000001:0.0:1761233851.469054:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233851.469057:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233851.469059:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233852.493043:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233852.493046:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233852.493049:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233853.517035:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233853.517038:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233853.517040:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233854.540991:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233854.540995:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233854.540997:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233855.565092:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233855.565095:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233855.565097:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233855.692943:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233855.692955:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233855.692957:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233855.692960:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233855.692962:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233855.692968:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233855.692974:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6324 00000100:00080000:1.0:1761233855.692977:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233855.692980:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.692983:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233855.692985:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.692988:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233855.692992:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233855.692993:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233855.692996:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233855.693009:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233855.693011:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233855.693014:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233855.693017:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693019:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233855.693020:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693022:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233855.693024:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233855.693028:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693030:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233855.693033:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233855.693036:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233855.693039:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693041:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233855.693043:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233855.693051:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761233855.693055:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233855.693059:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233855.693060:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693063:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233855.693066:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233855.693069:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233855.693077:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233855.693083:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233855.693090:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245219648/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233855.693099:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233855.693101:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233855.693105:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245219648/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233855.693125:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693127:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233855.693129:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233855.693133:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233855.693133:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233855.693135:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233855.693136:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233855.693138:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233855.693141:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6324 00000100:00080000:1.0:1761233855.693143:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233855.693146:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693149:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233855.693151:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693153:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233855.693155:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233855.693156:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233855.693158:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233855.693160:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233855.693164:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233855.693166:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233855.693182:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693186:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233855.693190:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693194:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233855.693199:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233855.693204:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693209:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233855.693212:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233855.693220:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233855.693226:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693229:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233855.693233:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233855.693238:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693242:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233855.693247:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761233855.693250:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233855.693255:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233855.693257:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693259:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233855.693261:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233855.693264:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233855.693270:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233855.693274:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233855.693281:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245219712/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233855.693293:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233855.693295:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233855.693301:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245219712/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233855.693324:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693328:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233855.693330:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233855.693336:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233855.693338:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233855.693341:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233855.693343:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233855.693346:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233855.693349:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6324 00000100:00080000:1.0:1761233855.693353:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233855.693358:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693360:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233855.693362:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693364:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233855.693366:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233855.693368:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233855.693371:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233855.693374:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233855.693376:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233855.693379:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233855.693385:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693390:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233855.693410:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693415:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233855.693417:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233855.693422:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233855.693428:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233855.693433:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233855.693436:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233855.693441:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693444:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233855.693446:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233855.693450:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693453:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233855.693458:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000800:00000200:2.1:1761233855.693461:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000020:00000040:1.0:1761233855.693463:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000800:00000200:2.1:1761233855.693465:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 02000000:00000001:1.0:1761233855.693468:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233855.693471:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693475:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233855.693477:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233855.693480:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233855.693487:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233855.693493:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233855.693500:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245219776/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233855.693523:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233855.693525:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233855.693533:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245219776/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233855.693554:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233855.693558:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233855.693562:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6334) 00000800:00000200:2.1:1761233855.693735:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233855.693736:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761233855.693908:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233855.693910:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000020:00000001:1.0:1761233856.589032:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233856.589036:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233856.589039:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233857.613037:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233857.613041:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233857.613044:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233858.637082:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233858.637085:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233858.637088:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233859.661155:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233859.661158:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233859.661160:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233860.685065:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233860.685069:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233860.685072:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233860.812984:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233860.812996:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233860.812999:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233860.813002:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233860.813005:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233860.813011:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233860.813018:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6329 00000100:00080000:1.0:1761233860.813020:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233860.813023:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813026:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233860.813028:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813029:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233860.813032:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233860.813034:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233860.813037:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233860.813040:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233860.813042:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233860.813043:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233860.813045:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813046:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233860.813047:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813048:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233860.813051:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233860.813054:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813055:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233860.813058:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233860.813061:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233860.813063:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813064:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233860.813066:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233860.813077:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761233860.813081:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233860.813086:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233860.813087:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813090:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233860.813094:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233860.813098:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233860.813106:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233860.813113:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233860.813120:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245219840/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233860.813129:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233860.813131:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233860.813134:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245219840/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233860.813151:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813153:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233860.813154:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233860.813157:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233860.813157:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233860.813158:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233860.813159:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233860.813161:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233860.813163:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6329 00000100:00080000:1.0:1761233860.813164:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233860.813165:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813166:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233860.813167:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813168:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233860.813169:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233860.813170:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233860.813170:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233860.813172:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233860.813173:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233860.813174:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233860.813175:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813176:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233860.813177:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813178:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233860.813180:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233860.813182:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813184:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233860.813185:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233860.813192:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233860.813193:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813195:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233860.813198:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233860.813202:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813204:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233860.813208:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761233860.813210:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233860.813212:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233860.813214:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813217:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233860.813218:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233860.813220:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233860.813223:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761233860.813226:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233860.813232:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245219904/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233860.813249:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233860.813250:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233860.813253:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245219904/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233860.813260:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813263:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233860.813264:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233860.813270:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233860.813271:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233860.813273:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233860.813274:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233860.813279:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233860.813282:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6329 00000100:00080000:1.0:1761233860.813284:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233860.813287:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813288:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233860.813289:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813290:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233860.813291:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233860.813293:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233860.813294:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233860.813296:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233860.813299:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233860.813300:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233860.813302:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813305:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233860.813306:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813307:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233860.813308:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233860.813309:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233860.813311:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233860.813338:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233860.813340:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233860.813344:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813345:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233860.813346:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233860.813348:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813350:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233860.813367:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761233860.813369:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233860.813371:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233860.813371:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813373:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233860.813375:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233860.813377:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233860.813381:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233860.813384:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233860.813388:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245219968/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233860.813397:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233860.813399:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233860.813404:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245219968/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233860.813418:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233860.813421:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233860.813424:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6339) 00000800:00000200:2.1:1761233860.813532:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233860.813535:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1761233860.813916:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233860.813918:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000800:00000200:2.1:1761233860.814151:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233860.814154:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 23 secs later) 00000020:00000001:0.0:1761233861.709055:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233861.709059:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233861.709063:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233862.733074:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233862.733079:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233862.733084:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233863.757040:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233863.757043:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233863.757046:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233864.781067:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233864.781074:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233864.781079:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233865.804990:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233865.804994:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233865.804998:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233865.933459:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233865.933463:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761233865.933696:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233865.933697:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761233865.933824:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233865.933825:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233866.829152:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233866.829156:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233866.829160:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233867.852987:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233867.852990:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233867.852992:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233868.877114:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233868.877120:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233868.877125:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233869.901046:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233869.901049:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233869.901051:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233870.925182:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233870.925185:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233870.925189:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233871.053593:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233871.053595:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761233871.053836:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233871.053837:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761233871.054005:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233871.054006:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000100:00000001:0.0:1761233871.564994:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233871.565000:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233871.565001:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233871.565006:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233871.565017:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233871.565019:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233871.565022:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233871.565024:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233871.565026:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761233871.949081:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233871.949084:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233871.949087:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233872.588978:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233872.588985:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233872.588986:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233872.588991:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233872.588996:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233872.589000:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233872.589002:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233872.589004:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233872.589006:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233872.973085:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233872.973089:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233872.973093:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233873.612941:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233873.612946:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233873.612947:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233873.612952:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233873.612955:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233873.612959:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233873.612961:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233873.612962:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233873.612964:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761233873.997039:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233873.997043:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233873.997045:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233874.636977:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233874.636981:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233874.636982:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233874.636985:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233874.636989:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233874.636991:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233874.636993:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233874.636995:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233874.636997:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233875.021035:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233875.021039:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233875.021041:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233875.660932:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233875.660936:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233875.660938:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233875.660941:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233875.660945:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233875.660948:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233875.660949:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233875.660951:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233875.660953:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761233876.045106:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233876.045110:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233876.045113:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233876.173523:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233876.173526:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761233876.173721:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233876.173724:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761233876.173978:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233876.173979:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1761233877.069190:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233877.069194:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233877.069198:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233878.093080:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233878.093084:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233878.093086:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233879.117177:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233879.117180:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233879.117184:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1761233880.141113:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233880.141118:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233880.141121:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233881.165096:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233881.165099:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233881.165102:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233881.293385:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233881.293387:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761233881.293590:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233881.293591:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761233881.293721:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233881.293722:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000020:00000001:0.0:1761233882.189116:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761233882.189121:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761233882.189124:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233883.213072:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233883.213076:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233883.213080:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233884.237084:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233884.237088:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233884.237092:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233885.261106:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233885.261110:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233885.261114:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233885.901046:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233885.901052:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233885.901054:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233885.901058:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233885.901063:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233885.901077:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233885.901080:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233885.901083:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233885.901086:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233886.285148:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233886.285151:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233886.285154:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233886.413625:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233886.413730:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233886.413770:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000800:00000200:3.1:1761233886.413931:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761233886.413969:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:0.0:1761233886.413999:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000100:00000001:0.0:1761233886.924980:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233886.924987:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233886.924989:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233886.924995:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233886.925001:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233886.925006:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233886.925009:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233886.925011:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233886.925014:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233887.309065:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233887.309069:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233887.309073:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233888.333156:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233888.333160:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233888.333164:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233889.357086:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233889.357090:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233889.357093:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1761233889.485349:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:0.0:1761233889.485417:0:88983:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1761233889.485429:0:88983:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:0.0:1761233889.485455:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233889.485460:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006f6738d0 (tot 1676778). 00000400:00000200:0.0:1761233889.485469:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233889.485475:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233889.485483:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b24b747f 00000400:00000010:0.0:1761233889.485486:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b24b747f. 00000100:00000001:0.0:1761233889.485492:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233889.485503:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@0000000025766f79 x1846784245220800/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233911 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233889.485555:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233889.485560:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000025766f79 x1846784245220800/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233911 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233889.485568:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233889.485571:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233889.485588:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233889.485591:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000eb8d1c51 (tot 1676530). 00000400:00000200:0.0:1761233889.485594:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233889.485597:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233889.485600:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cba99716 00000400:00000010:0.0:1761233889.485602:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cba99716. 00000100:00000001:0.0:1761233889.485604:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233889.485609:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000b4e0799e x1846784245220864/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233911 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233889.485617:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233889.485619:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000b4e0799e x1846784245220864/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233911 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233889.485625:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233889.485627:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761233889.485632:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761233889.485634:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000171ed136 (tot 1676282). 00000400:00000200:0.0:1761233889.485637:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761233889.485640:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761233889.485643:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000faee444f 00000400:00000010:0.0:1761233889.485645:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000faee444f. 00000100:00000001:0.0:1761233889.485647:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761233889.485651:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000009a39bc22 x1846784245220928/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233911 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233889.485659:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761233889.485661:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000009a39bc22 x1846784245220928/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233911 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233889.485673:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233889.485675:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000020:00000001:1.0:1761233890.381043:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233890.381047:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233890.381052:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233891.405026:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233891.405031:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233891.405035:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233891.533063:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233891.533079:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233891.533081:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233891.533085:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233891.533088:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233891.533095:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233891.533103:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6360 00000100:00080000:0.0:1761233891.533106:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233891.533110:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533113:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233891.533116:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533119:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233891.533123:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233891.533125:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233891.533129:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233891.533132:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233891.533134:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233891.533137:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233891.533140:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533141:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233891.533143:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533144:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233891.533146:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233891.533149:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533151:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233891.533154:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233891.533157:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233891.533162:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533164:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233891.533167:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233891.533175:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233891.533180:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233891.533186:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233891.533187:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533191:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233891.533195:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233891.533200:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233891.533210:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233891.533219:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233891.533229:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245220992/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233891.533240:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233891.533243:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233891.533248:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245220992/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233891.533279:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533282:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233891.533283:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233891.533287:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233891.533288:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233891.533290:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233891.533291:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233891.533294:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233891.533296:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6360 00000100:00080000:0.0:1761233891.533298:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233891.533301:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533303:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233891.533305:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533306:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233891.533309:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233891.533310:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233891.533312:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233891.533314:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233891.533315:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233891.533317:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233891.533320:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533322:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233891.533324:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533326:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233891.533328:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233891.533331:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533334:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233891.533336:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233891.533343:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233891.533346:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533348:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233891.533353:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233891.533358:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533360:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233891.533365:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233891.533369:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233891.533372:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233891.533374:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533377:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233891.533378:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233891.533380:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233891.533386:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233891.533392:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233891.533399:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245221056/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233891.533411:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233891.533414:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233891.533424:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245221056/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233891.533445:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533449:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233891.533452:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233891.533456:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233891.533459:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233891.533462:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233891.533464:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233891.533467:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233891.533471:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6360 00000100:00080000:0.0:1761233891.533475:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233891.533480:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533482:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233891.533485:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533489:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233891.533491:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233891.533493:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233891.533494:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233891.533496:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233891.533500:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233891.533503:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233891.533507:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533511:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233891.533517:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533520:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233891.533524:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233891.533529:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233891.533534:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233891.533538:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233891.533542:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233891.533546:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533550:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233891.533552:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233891.533556:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533560:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233891.533565:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233891.533569:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233891.533571:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233891.533572:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533574:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233891.533575:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233891.533576:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233891.533581:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233891.533585:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233891.533591:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245221120/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233891.533599:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233891.533601:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233891.533604:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245221120/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233891.533618:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233891.533620:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233891.533626:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6370) 00000800:00000200:3.1:1761233891.533694:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233891.533697:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233891.534225:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233891.534227:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761233891.534484:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233891.534486:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000020:00000001:1.0:1761233892.429062:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233892.429065:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233892.429067:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233893.453065:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233893.453070:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233893.453074:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233894.477073:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233894.477077:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233894.477082:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233895.501040:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233895.501043:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233895.501046:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233896.525119:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233896.525123:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233896.525126:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233896.653517:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233896.653519:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761233896.653708:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233896.653710:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761233896.653984:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233896.653985:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1761233897.549103:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233897.549108:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233897.549112:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233898.573039:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233898.573041:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233898.573044:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233899.597085:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233899.597089:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233899.597091:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233900.621029:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233900.621032:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233900.621034:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233901.645081:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233901.645083:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233901.645086:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233901.773065:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233901.773076:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233901.773077:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233901.773080:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233901.773082:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233901.773088:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233901.773094:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6370 00000100:00080000:0.0:1761233901.773098:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233901.773102:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773105:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233901.773109:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773111:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233901.773115:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233901.773116:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233901.773119:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233901.773122:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233901.773124:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233901.773127:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233901.773131:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773132:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233901.773134:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773136:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233901.773138:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233901.773141:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773143:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233901.773146:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233901.773149:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233901.773151:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773153:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233901.773155:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233901.773164:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000095969832. 00000020:00000040:0.0:1761233901.773167:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233901.773171:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233901.773173:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773176:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233901.773178:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233901.773182:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233901.773188:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233901.773194:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233901.773201:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000095969832 x1846784245221376/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233901.773210:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233901.773224:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233901.773227:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000095969832] to pc [ptlrpcd_rcv+-1] req@0000000095969832 x1846784245221376/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233901.773245:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773247:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233901.773249:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233901.773252:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233901.773253:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233901.773254:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233901.773255:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233901.773258:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233901.773260:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6370 00000100:00080000:0.0:1761233901.773262:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233901.773264:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773266:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233901.773267:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773269:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233901.773271:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233901.773272:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233901.773273:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233901.773275:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233901.773277:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233901.773279:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233901.773294:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773296:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233901.773299:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773305:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233901.773307:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233901.773311:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773314:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233901.773316:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233901.773321:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233901.773325:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773329:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233901.773332:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233901.773339:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773343:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233901.773348:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002ca01b78. 00000020:00000040:0.0:1761233901.773353:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233901.773359:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233901.773361:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773365:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233901.773368:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233901.773372:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233901.773381:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233901.773388:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233901.773396:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ca01b78 x1846784245221440/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233901.773407:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233901.773411:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233901.773418:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002ca01b78] to pc [ptlrpcd_rcv+-1] req@000000002ca01b78 x1846784245221440/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233901.773439:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773443:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233901.773445:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233901.773450:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233901.773451:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233901.773452:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233901.773454:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233901.773457:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233901.773463:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6370 00000100:00080000:0.0:1761233901.773467:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233901.773473:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773478:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233901.773485:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773488:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233901.773493:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233901.773496:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233901.773500:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233901.773504:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233901.773509:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233901.773522:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233901.773529:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773534:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1761233901.773537:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000040:0.0:1761233901.773539:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000800:00000200:2.1:1761233901.773541:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000100:00000001:0.0:1761233901.773545:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233901.773550:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233901.773558:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233901.773563:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233901.773568:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233901.773572:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233901.773577:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773581:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233901.773583:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233901.773590:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773593:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233901.773600:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233901.773606:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233901.773610:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233901.773613:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773619:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233901.773622:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233901.773625:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233901.773631:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233901.773637:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233901.773648:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245221504/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233901.773702:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233901.773705:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233901.773714:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245221504/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233901.773737:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233901.773743:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233901.773750:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6380) 00000800:00000200:2.1:1761233901.773932:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233901.773933:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233901.774073:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233901.774074:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1761233902.669114:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233902.669119:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233902.669122:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233903.693081:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233903.693084:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233903.693087:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233904.717119:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233904.717124:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233904.717130:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233905.741108:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233905.741113:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233905.741118:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233906.765069:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233906.765072:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233906.765075:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233906.893006:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233906.893024:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233906.893025:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233906.893029:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233906.893032:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233906.893039:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233906.893048:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6375 00000100:00080000:0.0:1761233906.893052:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233906.893056:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893059:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233906.893063:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893066:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233906.893070:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233906.893072:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233906.893076:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233906.893079:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233906.893082:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233906.893085:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233906.893089:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893093:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233906.893096:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893098:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233906.893101:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233906.893106:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893110:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233906.893113:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233906.893117:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233906.893120:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893122:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233906.893124:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233906.893134:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233906.893137:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233906.893142:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233906.893143:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893146:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233906.893149:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233906.893152:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233906.893160:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233906.893166:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233906.893174:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245221568/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233906.893184:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233906.893186:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233906.893190:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245221568/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233906.893210:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893213:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233906.893215:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233906.893219:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233906.893220:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233906.893221:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233906.893223:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233906.893226:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233906.893228:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6375 00000100:00080000:0.0:1761233906.893231:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233906.893233:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893236:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233906.893238:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893241:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233906.893243:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233906.893245:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233906.893247:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233906.893249:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233906.893251:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233906.893253:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233906.893257:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893261:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233906.893264:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893270:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233906.893275:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233906.893284:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893294:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233906.893302:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233906.893310:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233906.893317:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893326:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233906.893331:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233906.893338:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893344:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233906.893349:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233906.893354:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233906.893360:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233906.893364:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893367:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233906.893369:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233906.893374:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233906.893383:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233906.893392:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233906.893403:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245221632/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233906.893437:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233906.893440:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233906.893447:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245221632/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233906.893471:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893475:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233906.893477:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233906.893480:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233906.893481:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233906.893483:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233906.893485:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233906.893488:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233906.893495:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6375 00000100:00080000:0.0:1761233906.893499:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233906.893507:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893516:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233906.893525:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893535:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233906.893542:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233906.893545:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233906.893552:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233906.893556:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233906.893562:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233906.893568:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233906.893579:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000800:00000200:2.1:1761233906.893584:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:0.0:1761233906.893604:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1761233906.893604:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000100:00000040:0.0:1761233906.893612:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893618:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233906.893625:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233906.893634:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233906.893644:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233906.893650:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233906.893656:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233906.893662:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893673:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233906.893686:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233906.893695:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893700:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233906.893711:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233906.893719:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233906.893725:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233906.893729:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893736:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233906.893740:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233906.893748:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233906.893761:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233906.893772:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233906.893789:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245221696/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233906.893832:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233906.893834:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233906.893847:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245221696/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233906.893942:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233906.893947:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233906.893953:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6386) 00000800:00000200:2.1:1761233906.894129:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233906.894131:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761233906.894315:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233906.894316:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000020:00000001:1.0:1761233907.789086:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233907.789091:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233907.789096:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233908.812991:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233908.812994:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233908.812996:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233909.837069:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233909.837072:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233909.837074:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233910.861058:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233910.861061:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233910.861064:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233911.885129:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233911.885135:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233911.885140:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1761233912.013903:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233912.013906:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1761233912.014223:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233912.014225:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1761233912.014450:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233912.014453:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000020:00000001:1.0:1761233912.909008:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233912.909011:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233912.909014:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233913.933013:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233913.933019:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233913.933024:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233914.957065:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233914.957068:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233914.957072:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233915.981061:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233915.981065:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233915.981068:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233917.005093:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233917.005097:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233917.005101:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233917.133019:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233917.133037:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233917.133040:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233917.133043:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233917.133046:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233917.133054:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233917.133062:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6386 00000100:00080000:1.0:1761233917.133066:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233917.133070:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133073:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233917.133075:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133078:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233917.133083:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233917.133085:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233917.133088:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233917.133092:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233917.133094:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233917.133097:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233917.133100:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133102:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233917.133104:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133106:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233917.133109:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233917.133112:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133115:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233917.133119:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233917.133122:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233917.133125:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133127:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233917.133129:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233917.133139:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761233917.133143:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233917.133147:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233917.133148:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133151:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233917.133154:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233917.133158:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233917.133165:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761233917.133172:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233917.133179:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245221952/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233917.133187:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233917.133189:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233917.133192:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245221952/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233917.133213:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133214:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233917.133216:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233917.133219:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233917.133219:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233917.133220:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233917.133222:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233917.133224:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233917.133225:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6386 00000100:00080000:1.0:1761233917.133226:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233917.133228:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133229:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233917.133230:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133231:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233917.133232:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233917.133232:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233917.133233:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233917.133235:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233917.133236:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233917.133238:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233917.133241:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133244:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233917.133246:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133249:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233917.133251:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233917.133255:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133258:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233917.133260:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233917.133268:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233917.133271:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133273:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233917.133277:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233917.133284:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133291:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233917.133296:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761233917.133299:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233917.133303:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233917.133305:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133309:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233917.133310:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233917.133312:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233917.133323:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761233917.133331:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233917.133341:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245222016/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233917.133363:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233917.133365:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233917.133370:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245222016/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233917.133383:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133386:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233917.133392:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233917.133399:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233917.133401:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233917.133403:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233917.133405:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233917.133409:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233917.133413:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6386 00000100:00080000:1.0:1761233917.133416:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233917.133421:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133425:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233917.133446:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133451:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233917.133455:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233917.133456:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233917.133458:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233917.133461:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233917.133465:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233917.133470:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233917.133479:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133493:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233917.133499:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133503:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233917.133510:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233917.133515:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233917.133516:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233917.133517:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233917.133521:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233917.133523:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133528:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233917.133530:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233917.133535:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133541:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233917.133547:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000020:00000040:1.0:1761233917.133553:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233917.133558:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233917.133560:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233917.133568:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233917.133571:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233917.133574:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233917.133581:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761233917.133590:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233917.133601:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245222080/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233917.133612:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233917.133614:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233917.133621:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245222080/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233917.133638:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761233917.133638:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:1.0:1761233917.133645:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:3.1:1761233917.133648:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000100:00000040:1.0:1761233917.133652:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6396) 00000800:00000200:3.1:1761233917.133905:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233917.133906:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761233917.134046:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233917.134048:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000020:00000001:1.0:1761233918.029106:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233918.029111:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233918.029115:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233919.053097:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233919.053100:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233919.053103:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233920.077017:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233920.077020:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233920.077022:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233921.101070:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233921.101074:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233921.101077:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233922.125079:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233922.125086:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233922.125092:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233922.252990:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233922.253004:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233922.253006:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233922.253010:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233922.253013:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233922.253020:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233922.253029:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6391 00000100:00080000:1.0:1761233922.253033:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233922.253037:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253040:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233922.253042:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253045:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233922.253050:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233922.253052:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233922.253055:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233922.253058:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233922.253061:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233922.253064:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233922.253067:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253069:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233922.253071:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253073:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233922.253076:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233922.253080:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253083:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233922.253086:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233922.253090:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233922.253093:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253095:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233922.253098:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233922.253111:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761233922.253127:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233922.253137:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233922.253138:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253142:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233922.253145:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233922.253150:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233922.253159:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761233922.253167:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233922.253176:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245222144/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233922.253187:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233922.253189:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233922.253194:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245222144/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233922.253218:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253221:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233922.253224:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233922.253228:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233922.253229:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233922.253230:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233922.253231:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233922.253234:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233922.253237:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6391 00000100:00080000:1.0:1761233922.253239:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233922.253242:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253245:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233922.253247:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253249:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233922.253252:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233922.253253:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233922.253255:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233922.253257:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233922.253260:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233922.253265:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233922.253270:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253274:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233922.253279:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253284:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233922.253289:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233922.253294:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253298:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233922.253302:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233922.253311:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233922.253317:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253320:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233922.253324:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233922.253331:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253334:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233922.253339:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761233922.253360:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233922.253364:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233922.253367:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253370:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233922.253371:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233922.253373:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233922.253381:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233922.253388:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233922.253398:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245222208/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233922.253429:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233922.253432:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233922.253441:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245222208/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233922.253461:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253464:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233922.253466:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233922.253470:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233922.253471:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233922.253473:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233922.253475:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233922.253481:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233922.253484:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6391 00000100:00080000:1.0:1761233922.253487:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233922.253492:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253495:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233922.253497:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253499:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233922.253504:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233922.253507:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233922.253511:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233922.253514:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233922.253519:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233922.253524:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233922.253532:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253537:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233922.253542:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253546:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233922.253549:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233922.253554:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233922.253560:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233922.253566:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233922.253570:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.1:1761233922.253575:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000001:1.0:1761233922.253576:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761233922.253580:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000100:00000001:1.0:1761233922.253581:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233922.253585:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233922.253590:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253594:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233922.253598:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761233922.253603:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233922.253608:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233922.253611:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253615:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233922.253617:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233922.253621:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233922.253626:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233922.253632:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233922.253642:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245222272/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233922.253652:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233922.253655:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233922.253662:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245222272/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233922.253693:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233922.253697:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233922.253705:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6401) 00000800:00000200:3.1:1761233922.253926:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233922.253927:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761233922.254096:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233922.254112:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000020:00000001:1.0:1761233923.149071:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233923.149074:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233923.149076:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233924.173042:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233924.173045:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233924.173047:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233925.197064:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233925.197067:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233925.197069:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233926.220980:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233926.220983:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233926.220986:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233927.245087:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233927.245090:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233927.245093:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761233927.372983:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233927.372992:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233927.372994:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761233927.372996:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233927.372997:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233927.373001:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233927.373007:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6396 00000100:00080000:1.0:1761233927.373009:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233927.373011:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373014:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233927.373016:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373017:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233927.373020:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233927.373022:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761233927.373024:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233927.373026:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761233927.373028:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761233927.373030:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761233927.373031:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373033:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233927.373035:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373036:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233927.373038:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761233927.373041:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373043:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233927.373045:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233927.373047:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233927.373049:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373051:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761233927.373052:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233927.373060:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f8a85602. 00000020:00000040:1.0:1761233927.373063:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761233927.373066:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233927.373067:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373070:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233927.373072:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233927.373075:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233927.373081:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761233927.373086:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233927.373091:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f8a85602 x1846784245222336/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233927.373097:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233927.373099:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233927.373101:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f8a85602] to pc [ptlrpcd_rcv+-1] req@00000000f8a85602 x1846784245222336/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233927.373114:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373115:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233927.373116:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233927.373118:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233927.373119:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761233927.373120:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233927.373120:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233927.373122:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233927.373123:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6396 00000100:00080000:1.0:1761233927.373124:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233927.373126:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373127:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233927.373127:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373128:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233927.373129:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233927.373130:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761233927.373131:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233927.373132:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761233927.373133:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761233927.373135:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761233927.373136:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373137:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233927.373138:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373139:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233927.373140:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761233927.373142:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373143:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233927.373143:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233927.373147:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233927.373148:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373149:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233927.373151:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233927.373153:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373154:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233927.373157:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761233927.373158:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761233927.373160:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233927.373160:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373161:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233927.373162:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233927.373163:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233927.373165:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761233927.373167:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233927.373169:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245222400/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233927.373173:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233927.373174:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233927.373175:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245222400/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233927.373184:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373186:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761233927.373187:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761233927.373190:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761233927.373191:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761233927.373191:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761233927.373192:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761233927.373193:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761233927.373195:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6396 00000100:00080000:1.0:1761233927.373197:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761233927.373199:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373200:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233927.373201:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373202:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761233927.373203:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761233927.373203:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761233927.373204:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761233927.373205:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761233927.373206:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761233927.373207:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761233927.373209:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373210:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761233927.373212:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373214:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761233927.373215:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761233927.373216:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761233927.373218:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761233927.373219:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761233927.373220:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761233927.373221:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373222:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761233927.373224:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761233927.373225:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373226:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761233927.373229:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761233927.373231:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761233927.373232:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761233927.373233:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373235:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761233927.373237:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761233927.373237:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761233927.373239:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761233927.373242:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761233927.373245:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245222464/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233927.373250:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761233927.373253:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761233927.373260:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245222464/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761233927.373304:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761233927.373307:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761233927.373312:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6406) 00000800:00000200:3.1:1761233927.373465:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233927.373468:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233927.373680:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233927.373682:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761233927.373955:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233927.373957:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1761233928.269002:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233928.269006:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233928.269009:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233929.293064:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233929.293067:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233929.293069:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233930.317152:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233930.317157:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233930.317162:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233931.341063:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233931.341065:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233931.341069:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233932.365137:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233932.365141:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233932.365145:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233932.493084:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233932.493099:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233932.493102:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233932.493106:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233932.493109:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233932.493116:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233932.493126:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6401 00000100:00080000:0.0:1761233932.493130:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233932.493135:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493140:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233932.493143:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493146:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233932.493152:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233932.493155:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233932.493159:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233932.493162:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233932.493166:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233932.493170:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233932.493175:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493178:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233932.493181:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493184:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233932.493210:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233932.493231:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493242:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233932.493246:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233932.493249:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233932.493252:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.493255:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233932.493258:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233932.493269:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233932.493274:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233932.493279:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233932.493281:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.493284:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233932.493287:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233932.493291:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233932.493302:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233932.493309:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233932.493316:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245222528/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233932.493328:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233932.493330:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233932.493334:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245222528/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233932.493362:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.493365:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233932.493367:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233932.493372:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233932.493373:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233932.493374:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233932.493376:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233932.493379:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233932.493382:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6401 00000100:00080000:0.0:1761233932.493384:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233932.493387:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493389:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233932.493391:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493393:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233932.493396:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233932.493397:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233932.493399:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233932.493401:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233932.493404:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233932.493407:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233932.493417:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493419:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233932.493421:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493423:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233932.493426:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233932.493429:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493431:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233932.493433:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233932.493439:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233932.493442:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.493445:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233932.493449:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233932.493454:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.493456:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233932.493461:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233932.493464:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233932.493467:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233932.493471:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.493475:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233932.493478:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233932.493482:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233932.493490:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233932.493497:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233932.493508:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245222592/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233932.493533:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233932.493536:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233932.493547:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245222592/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233932.493566:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.493572:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233932.493576:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233932.493585:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233932.493590:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233932.493594:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233932.493597:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233932.493605:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233932.493613:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6401 00000100:00080000:0.0:1761233932.493621:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233932.493629:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493637:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233932.493643:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493646:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233932.493649:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233932.493651:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233932.493655:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233932.493659:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233932.493663:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233932.493669:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233932.493680:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493688:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233932.493696:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493703:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233932.493710:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233932.493720:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233932.493727:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233932.493735:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233932.493741:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.1:1761233932.493742:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000001:0.0:1761233932.493748:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761233932.493749:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1761233932.493755:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233932.493761:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233932.493783:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.493787:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233932.493797:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233932.493806:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233932.493816:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233932.493820:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.493828:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233932.493831:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233932.493834:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233932.493843:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233932.493853:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233932.493924:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245222656/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233932.493978:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233932.493983:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233932.493997:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245222656/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233932.494042:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233932.494048:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233932.494056:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6411) 00000800:00000200:3.1:1761233932.494310:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233932.494311:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761233932.494485:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233932.494486:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233933.389047:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233933.389050:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233933.389052:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233934.413211:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233934.413216:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233934.413222:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233935.437042:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233935.437045:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233935.437048:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233936.077028:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233936.077033:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233936.077035:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233936.077038:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233936.077043:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233936.077046:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233936.077048:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233936.077049:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233936.077051:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233936.461150:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233936.461153:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233936.461156:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233937.100944:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233937.100949:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233937.100950:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233937.100953:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233937.100957:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233937.100967:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233937.100969:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233937.100971:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233937.100972:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233937.485081:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233937.485085:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233937.485089:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233937.613439:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233937.613443:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761233937.613675:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233937.613676:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761233937.613805:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233937.613807:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000100:00000001:0.0:1761233938.124926:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233938.124930:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233938.124931:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233938.124934:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233938.124938:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233938.124942:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233938.124944:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233938.124947:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233938.124949:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233938.509055:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233938.509057:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233938.509060:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233939.149020:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233939.149026:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233939.149028:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233939.149034:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233939.149038:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233939.149042:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233939.149044:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233939.149046:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233939.149048:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233939.533275:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233939.533279:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233939.533283:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233940.172928:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233940.172933:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233940.172934:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233940.172938:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233940.172942:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233940.172945:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233940.172947:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233940.172949:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233940.172951:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233940.557112:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233940.557115:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233940.557117:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233941.197106:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233941.197111:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233941.197113:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233941.197118:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233941.197123:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233941.197126:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233941.197128:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233941.197132:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233941.197134:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233941.581064:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233941.581067:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233941.581069:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233942.220898:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233942.220902:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233942.220903:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233942.220907:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233942.220911:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233942.220914:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233942.220915:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233942.220917:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233942.220919:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233942.605076:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233942.605079:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233942.605082:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:2.1:1761233942.733423:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233942.733428:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000100:00000001:3.0:1761233942.733513:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233942.733517:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233942.733520:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233942.733526:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000296782d7. 00000100:00000001:3.0:1761233942.733532:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233942.733542:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ecb9068 x1846784245223040/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233942.733571:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233942.733572:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233942.733580:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ecb9068] to pc [ptlrpcd_rcv+-1] req@000000009ecb9068 x1846784245223040/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233942.733600:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233942.733604:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:3.0:1761233942.733609:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6421) 00000800:00000200:2.1:1761233942.733735:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233942.733737:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761233942.733900:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233942.733901:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000100:00000001:0.0:1761233943.244960:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233943.244965:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233943.244966:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233943.244971:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233943.244976:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233943.244979:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233943.244981:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233943.244983:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233943.244985:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233943.629055:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233943.629059:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233943.629063:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233944.653057:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233944.653060:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233944.653064:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233945.677013:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233945.677017:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233945.677019:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233946.701089:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233946.701092:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233946.701094:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233947.725015:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233947.725018:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233947.725021:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233947.853020:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233947.853028:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233947.853030:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233947.853032:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233947.853034:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233947.853038:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233947.853043:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6416 00000100:00080000:0.0:1761233947.853046:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233947.853048:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853050:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233947.853052:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853053:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233947.853056:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233947.853057:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233947.853059:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233947.853063:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233947.853064:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233947.853066:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233947.853068:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853069:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233947.853070:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853071:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233947.853072:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233947.853075:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853076:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233947.853078:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233947.853081:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233947.853083:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853084:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233947.853086:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233947.853094:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233947.853096:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233947.853099:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233947.853100:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853102:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233947.853105:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233947.853108:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233947.853114:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233947.853118:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233947.853124:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245223104/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233947.853131:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233947.853133:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233947.853136:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245223104/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233947.853153:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853155:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233947.853157:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233947.853159:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233947.853160:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233947.853161:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233947.853162:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233947.853163:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233947.853165:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6416 00000100:00080000:0.0:1761233947.853166:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233947.853167:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853173:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233947.853174:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853175:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233947.853176:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233947.853176:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233947.853177:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233947.853178:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233947.853179:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233947.853181:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233947.853182:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853184:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233947.853185:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853186:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233947.853187:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233947.853188:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853189:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233947.853191:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233947.853195:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233947.853196:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853197:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233947.853200:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233947.853204:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853205:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233947.853218:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233947.853220:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233947.853222:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233947.853223:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853225:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233947.853226:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233947.853228:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233947.853231:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233947.853234:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233947.853240:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245223168/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233947.853247:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233947.853249:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233947.853254:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245223168/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233947.853270:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853273:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233947.853274:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233947.853277:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233947.853278:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233947.853279:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233947.853281:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233947.853283:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233947.853286:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6416 00000100:00080000:0.0:1761233947.853288:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233947.853292:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853296:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233947.853298:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853301:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233947.853305:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233947.853306:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233947.853308:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233947.853309:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233947.853311:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233947.853314:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233947.853316:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853319:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233947.853321:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853323:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233947.853324:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233947.853326:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233947.853330:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233947.853334:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233947.853336:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233947.853337:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853338:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233947.853340:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233947.853342:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853344:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233947.853347:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233947.853350:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233947.853351:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233947.853352:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853352:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233947.853354:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233947.853355:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233947.853360:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233947.853364:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233947.853370:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245223232/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233947.853384:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233947.853386:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233947.853391:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245223232/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233947.853418:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233947.853422:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233947.853426:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6426) 00000800:00000200:2.1:1761233947.853500:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233947.853519:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761233947.853737:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233947.853739:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761233947.854014:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233947.854015:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000020:00000001:1.0:1761233948.749125:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233948.749128:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233948.749131:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233949.773257:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233949.773262:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233949.773267:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233950.413092:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233950.413099:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233950.413102:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233950.413109:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233950.413115:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233950.413119:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233950.413123:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233950.413126:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233950.413130:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233950.797117:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233950.797120:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233950.797122:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233951.436992:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233951.436997:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233951.436998:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233951.437002:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233951.437006:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233951.437008:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233951.437011:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233951.437013:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233951.437015:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233951.821125:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233951.821128:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233951.821131:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233952.460992:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233952.460996:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233952.460998:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233952.461002:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233952.461006:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233952.461008:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233952.461010:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233952.461011:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233952.461013:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233952.845097:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233952.845103:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233952.845109:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233952.972966:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233952.972978:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233952.972980:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233952.972984:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233952.972987:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233952.972993:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233952.973000:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6421 00000100:00080000:0.0:1761233952.973002:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233952.973005:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973007:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233952.973009:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973011:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233952.973014:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233952.973016:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233952.973018:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233952.973021:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233952.973023:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233952.973026:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233952.973028:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973030:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233952.973032:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973033:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233952.973035:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233952.973038:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973040:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233952.973043:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233952.973046:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233952.973048:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233952.973050:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233952.973051:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233952.973062:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761233952.973064:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233952.973069:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233952.973070:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233952.973074:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233952.973077:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233952.973082:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233952.973090:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233952.973097:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233952.973120:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233952.973150:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233952.973152:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233952.973157:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233952.973176:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233952.973178:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233952.973180:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233952.973183:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233952.973184:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233952.973185:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233952.973186:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233952.973188:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233952.973189:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6421 00000100:00080000:0.0:1761233952.973191:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233952.973192:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973194:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233952.973195:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973196:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233952.973199:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233952.973201:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233952.973202:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233952.973204:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233952.973207:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233952.973209:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233952.973212:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973215:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233952.973217:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973219:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233952.973220:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233952.973222:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973225:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233952.973228:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233952.973235:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233952.973236:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233952.973238:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233952.973242:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233952.973244:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233952.973246:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973249:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1761233952.973250:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233952.973257:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:3.0:1761233952.973260:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1761233952.973261:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233952.973263:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973264:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1761233952.973269:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233952.973270:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233952.973273:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233952.973277:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000040:3.0:1761233952.973279:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233952.973282:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233952.973287:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233952.973334:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233952.973335:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233952.973343:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233952.973358:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761233952.973360:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233952.973369:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1761233952.973371:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233952.973373:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233952.973375:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1761233952.973375:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233952.973381:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233952.973383:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233952.973385:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1761233952.973386:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eff3cf06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245223296:192.168.201.124@tcp:250: 00000100:00080000:0.0:1761233952.973387:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233952.973390:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233952.973395:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6421 00000100:00080000:0.0:1761233952.973398:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1761233952.973399:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1761233952.973402:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973404:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233952.973405:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 02000000:00000001:3.0:1761233952.973406:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1761233952.973407:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233952.973410:0:64357:0:(genops.c:899:class_conn2export()) Process entered 02000000:00000001:3.0:1761233952.973411:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1761233952.973412:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233952.973413:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233952.973414:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233952.973417:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:3.0:1761233952.973418:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1761233952.973419:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233952.973422:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973423:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233952.973424:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233952.973425:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233952.973426:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233952.973427:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 02000000:00000010:3.0:1761233952.973429:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000296782d7. 00000100:00000001:0.0:1761233952.973430:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233952.973434:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233952.973435:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233952.973438:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233952.973442:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761233952.973443:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00000001:0.0:1761233952.973444:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233952.973448:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233952.973450:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761233952.973451:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000001:0.0:1761233952.973453:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233952.973457:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000100:00000200:3.0:1761233952.973458:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245223296, portal 25 00000020:00000040:0.0:1761233952.973460:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000001:3.0:1761233952.973462:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1761233952.973465:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:3.0:1761233952.973466:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246294528 : -104717463257088 : ffffa0c290cd0000) 02000000:00000001:0.0:1761233952.973470:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233952.973474:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233952.973476:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233952.973478:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233952.973479:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761233977 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 02000000:00000010:0.0:1761233952.973484:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:3.0:1761233952.973487:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:0.0:1761233952.973488:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233952.973492:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00100000:0.0:1761233952.973495:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:3.0:1761233952.973498:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000008d1459c3. 00000100:00000200:3.0:1761233952.973503:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245223296, offset 0 00000100:00000001:0.0:1761233952.973511:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233952.973512:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761233952.973518:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000040:0.0:1761233952.973520:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233952.973528:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233952.973529:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233952.973533:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6432) 00000400:00000200:3.0:1761233952.973535:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761233952.973543:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761233952.973546:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761233952.973549:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761233952.973552:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11706:11706:256:4294967295] 192.168.201.124@tcp LPNI seq info [11706:11706:8:4294967295] 00000400:00000200:3.0:1761233952.973560:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761233952.973565:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761233952.973568:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000171ed136 (tot 1676530). 00000800:00000200:3.0:1761233952.973574:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233952.973577:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761233952.973596:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973597:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761233952.973599:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973601:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973603:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233952.973609:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761233952.973611:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1761233952.973613:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761233952.973617:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761233952.973619:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761233952.973623:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233952.973629:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761233952.973630:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233952.973631:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761233952.973633:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233952.973635:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c218247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245223360:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761233952.973637:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761233952.973639:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761233952.973639:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233952.973641:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761233952.973643:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000be06bfbf. 02000000:00000001:3.0:1761233952.973644:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761233952.973648:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761233952.973650:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761233952.973651:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245223360, portal 10 00000100:00000001:3.0:1761233952.973654:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761233952.973656:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246306688 : -104717463244928 : ffffa0c290cd2f80) 00000100:00000040:3.0:1761233952.973664:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761233977 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233952.973668:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761233952.973670:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761233952.973671:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000046716e3e. 00000100:00000200:3.0:1761233952.973672:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245223360, offset 0 00000400:00000200:3.0:1761233952.973675:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761233952.973678:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761233952.973680:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761233952.973682:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761233952.973685:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761233952.973690:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11707:11707:255:4294967295] 192.168.201.124@tcp LPNI seq info [11707:11707:7:4294967295] 00000400:00000200:3.0:1761233952.973696:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761233952.973698:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761233952.973701:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000eb8d1c51 (tot 1676778). 00000800:00000200:3.0:1761233952.973703:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233952.973710:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761233952.973722:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973723:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761233952.973725:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973727:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761233952.973731:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233952.973735:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761233952.973736:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233952.973737:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761233952.973738:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233952.973740:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070caacfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245223424:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761233952.973741:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761233952.973742:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761233952.973744:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233952.973746:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761233952.973749:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062959873. 02000000:00000001:3.0:1761233952.973751:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761233952.973753:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000009aaa4f6b. 00000400:00000010:3.0:1761233952.973758:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761233952.973763:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245223424, portal 4 00000100:00000001:3.0:1761233952.973765:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761233952.973765:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246301824 : -104717463249792 : ffffa0c290cd1c80) 00000100:00000040:3.0:1761233952.973771:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761233977 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233952.973776:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761233952.973777:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761233952.973780:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000e466389f. 00000100:00000200:3.0:1761233952.973781:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245223424, offset 0 00000400:00000200:3.0:1761233952.973796:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761233952.973799:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761233952.973802:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761233952.973806:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761233952.973809:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761233952.973812:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11708:11708:254:4294967295] 192.168.201.124@tcp LPNI seq info [11708:11708:6:4294967295] 00000400:00000200:3.0:1761233952.973827:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761233952.973837:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761233952.973841:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006f6738d0 (tot 1677026). 00000800:00000200:3.0:1761233952.973900:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761233952.973903:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761233952.973906:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973907:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761233952.973909:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973914:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973916:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233952.973919:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761233952.973921:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1761233952.973925:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761233952.973927:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761233952.973933:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233952.973934:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233953.484967:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233953.484974:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233953.484976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233953.484981:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233953.484988:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233953.484992:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233953.484996:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233953.484998:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233953.485001:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233953.869114:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233953.869118:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233953.869123:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233954.509012:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233954.509019:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233954.509021:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233954.509027:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233954.509034:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233954.509039:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233954.509041:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233954.509043:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233954.509046:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233954.893014:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233954.893019:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233954.893022:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233955.533000:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233955.533006:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233955.533007:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233955.533012:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233955.533017:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233955.533022:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233955.533024:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233955.533027:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233955.533030:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233955.917091:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233955.917094:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233955.917098:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761233956.045499:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761233956.045507:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761233956.045512:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761233956.045522:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233952/real 1761233956] req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761233956.045554:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761233956.045557:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761233956.045560:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761233956.045565:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761233956.045566:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761233956.045569:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761233956.045573:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761233956.045581:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233956.045588:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761233956.045590:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045593:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761233956.045596:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761233956.045599:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233956.045606:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761233956.045611:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761233956.045619:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761233956.045623:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045625:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761233956.045625:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045629:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761233956.045631:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761233956.045636:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761233956.045642:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761233956.045646:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761233956.045650:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761233956.045659:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c218247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245223360:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761233956.045665:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761233956.045668:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233952/real 1761233956] req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761233956.045677:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761233956.045678:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761233956.045680:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000009aaa4f6b. 00000400:00000200:3.0:1761233956.045682:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761233956.045684:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761233956.045685:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761233956.045688:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761233956.045696:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233956.045702:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761233956.045703:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045705:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761233956.045706:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761233956.045709:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233956.045714:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761233956.045717:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761233956.045722:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761233956.045724:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045725:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761233956.045726:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045728:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761233956.045729:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761233956.045731:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761233956.045734:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761233956.045736:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761233956.045739:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761233956.045746:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070caacfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245223424:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761233956.045750:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761233956.045752:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761233952/real 1761233956] req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761233956.045759:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761233956.045760:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761233956.045762:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761233956.045764:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761233956.045766:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761233956.045768:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761233956.045769:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761233956.045781:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233956.045786:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761233956.045787:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045789:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761233956.045790:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761233956.045793:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233956.045798:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761233956.045801:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761233956.045806:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761233956.045808:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045809:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761233956.045811:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045812:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761233956.045813:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761233956.045834:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761233956.045837:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761233956.045839:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761233956.045842:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761233956.045893:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eff3cf06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245223296:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761233956.045898:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045900:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761233956.045903:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000003c218247 x1846784245223360/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761233956.045909:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761233956.045912:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761233956.045917:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000be06bfbf. 02000000:00000001:3.0:1761233956.045923:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761233956.045926:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761233956.045928:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761233956.045931:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761233956.045934:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000040aa0f54. 02000000:00000001:3.0:1761233956.045939:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761233956.045941:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761233956.045942:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000003c218247. 00000100:00000001:3.0:1761233956.045947:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761233956.045948:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045950:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761233956.045953:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000070caacfc x1846784245223424/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761233956.045959:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761233956.045960:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761233956.045961:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062959873. 02000000:00000001:3.0:1761233956.045965:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761233956.045966:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761233956.045967:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761233956.045969:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761233956.045970:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078138607. 02000000:00000001:3.0:1761233956.045973:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761233956.045974:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761233956.045975:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000070caacfc. 00000100:00000001:3.0:1761233956.045977:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761233956.045978:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.045980:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761233956.045982:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000eff3cf06 x1846784245223296/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761233977 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761233956.045987:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761233956.045988:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761233956.045989:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000296782d7. 02000000:00000001:3.0:1761233956.045993:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761233956.045995:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761233956.045996:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761233956.045997:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761233956.045999:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:3.0:1761233956.046001:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761233956.046002:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761233956.046003:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000eff3cf06. 00000100:00000001:3.0:1761233956.046006:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761233956.046007:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.046009:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761233956.046022:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761233956.046024:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233956.046026:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761233956.046028:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233956.556979:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233956.556984:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233956.556985:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233956.556989:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233956.556992:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233956.556995:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233956.556997:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233956.556998:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233956.557000:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233956.941084:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233956.941088:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233956.941092:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233957.580948:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233957.580953:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233957.580955:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233957.580960:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233957.580965:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233957.580969:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233957.580971:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233957.580974:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233957.580976:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233957.965100:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233957.965104:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233957.965108:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233958.093026:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761233958.093036:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761233958.093037:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761233958.093039:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761233958.093041:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761233958.093045:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761233958.093051:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6427 00000100:00080000:3.0:1761233958.093053:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761233958.093055:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093057:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233958.093058:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093060:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761233958.093062:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761233958.093064:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761233958.093066:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761233958.093068:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761233958.093069:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761233958.093071:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761233958.093072:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093074:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233958.093075:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093076:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761233958.093078:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761233958.093081:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093082:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233958.093085:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233958.093087:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761233958.093089:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093090:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761233958.093091:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761233958.093099:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ecb9068. 00000020:00000040:3.0:1761233958.093101:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761233958.093104:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761233958.093106:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093108:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233958.093110:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233958.093113:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233958.093120:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000296782d7. 00000100:00000001:3.0:1761233958.093126:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233958.093131:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ecb9068 x1846784245223488/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233958.093138:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233958.093139:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233958.093142:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ecb9068] to pc [ptlrpcd_rcv+-1] req@000000009ecb9068 x1846784245223488/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233958.093164:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093165:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761233958.093167:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761233958.093169:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761233958.093169:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761233958.093170:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761233958.093171:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761233958.093173:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761233958.093174:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6427 00000100:00080000:3.0:1761233958.093176:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761233958.093178:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093179:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233958.093180:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093181:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761233958.093183:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761233958.093184:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761233958.093185:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761233958.093186:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761233958.093189:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761233958.093190:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761233958.093193:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093194:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233958.093196:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093198:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761233958.093199:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761233958.093202:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093204:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233958.093205:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233958.093211:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761233958.093212:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093214:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761233958.093217:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761233958.093220:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093222:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761233958.093225:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000194cd3a7. 00000020:00000040:3.0:1761233958.093227:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761233958.093230:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761233958.093231:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093233:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233958.093234:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233958.093236:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233958.093239:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000062959873. 00000100:00000001:3.0:1761233958.093244:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233958.093248:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000194cd3a7 x1846784245223552/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233958.093256:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233958.093258:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233958.093262:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000194cd3a7] to pc [ptlrpcd_rcv+-1] req@00000000194cd3a7 x1846784245223552/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233958.093279:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093283:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761233958.093285:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761233958.093291:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761233958.093293:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761233958.093295:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761233958.093297:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761233958.093301:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761233958.093307:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6427 00000100:00080000:3.0:1761233958.093312:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761233958.093316:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093320:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233958.093324:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093328:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761233958.093331:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761233958.093332:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761233958.093334:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761233958.093337:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:3.0:1761233958.093342:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:3.0:1761233958.093345:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:3.0:1761233958.093349:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093352:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233958.093354:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093357:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761233958.093359:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:3.0:1761233958.093362:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761233958.093366:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233958.093367:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233958.093371:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761233958.093375:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093378:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761233958.093380:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761233958.093384:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093386:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761233958.093392:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a52d2576. 00000020:00000040:3.0:1761233958.093395:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:3.0:1761233958.093400:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761233958.093402:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093405:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233958.093407:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233958.093410:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233958.093415:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000be06bfbf. 00000100:00000001:3.0:1761233958.093420:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233958.093426:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a52d2576 x1846784245223616/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233958.093439:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233958.093440:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233958.093443:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a52d2576] to pc [ptlrpcd_rcv+-1] req@00000000a52d2576 x1846784245223616/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233958.093449:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233958.093451:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:3.0:1761233958.093456:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6437) 00000800:00000200:2.1:1761233958.093518:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233958.093521:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233958.093901:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233958.093903:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761233958.094123:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233958.094125:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1761233958.604962:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233958.604966:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233958.604967:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233958.604970:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233958.604975:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233958.604979:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233958.604980:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233958.604983:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233958.604984:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233958.989051:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233958.989054:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233958.989057:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233959.628926:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233959.628931:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233959.628932:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233959.628936:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233959.628940:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233959.628943:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233959.628945:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233959.628947:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233959.628949:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233960.013125:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233960.013132:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233960.013139:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761233960.652971:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761233960.652979:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761233960.652981:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233960.652988:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761233960.653011:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761233960.653016:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233960.653020:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761233960.653023:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761233960.653026:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761233961.037052:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233961.037055:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233961.037057:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233962.061058:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233962.061061:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233962.061064:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233963.085126:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233963.085130:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233963.085134:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233963.213025:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761233963.213038:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761233963.213041:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761233963.213045:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761233963.213048:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761233963.213054:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761233963.213062:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6432 00000100:00080000:3.0:1761233963.213065:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761233963.213069:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213072:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233963.213075:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213077:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761233963.213081:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761233963.213083:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761233963.213087:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761233963.213090:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761233963.213093:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761233963.213096:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761233963.213099:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213101:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233963.213103:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213105:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761233963.213108:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761233963.213112:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213114:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233963.213117:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233963.213121:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761233963.213123:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213125:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761233963.213127:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761233963.213155:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000266eaa93. 00000020:00000040:3.0:1761233963.213159:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761233963.213164:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761233963.213165:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213169:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233963.213172:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233963.213177:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233963.213186:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074a27ae0. 00000100:00000001:3.0:1761233963.213193:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233963.213202:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000266eaa93 x1846784245223680/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233963.213214:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233963.213216:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233963.213221:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000266eaa93] to pc [ptlrpcd_rcv+-1] req@00000000266eaa93 x1846784245223680/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233963.213249:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213252:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761233963.213254:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761233963.213259:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761233963.213260:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761233963.213262:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761233963.213263:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761233963.213266:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761233963.213269:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6432 00000100:00080000:3.0:1761233963.213271:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761233963.213273:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213275:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233963.213277:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213279:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761233963.213281:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761233963.213283:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761233963.213285:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761233963.213287:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761233963.213290:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761233963.213292:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761233963.213295:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213299:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233963.213304:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213315:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761233963.213319:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761233963.213323:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213329:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233963.213335:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233963.213344:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761233963.213350:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213356:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761233963.213360:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761233963.213367:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213373:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761233963.213379:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004b242e46. 00000020:00000040:3.0:1761233963.213387:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761233963.213393:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761233963.213397:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213403:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233963.213406:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233963.213410:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233963.213418:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074003a94. 00000100:00000001:3.0:1761233963.213424:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233963.213436:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004b242e46 x1846784245223744/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233963.213450:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233963.213454:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233963.213461:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004b242e46] to pc [ptlrpcd_rcv+-1] req@000000004b242e46 x1846784245223744/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233963.213476:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213480:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761233963.213483:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761233963.213492:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761233963.213497:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761233963.213502:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761233963.213505:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761233963.213510:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761233963.213518:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6432 00000100:00080000:3.0:1761233963.213524:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761233963.213531:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213537:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233963.213544:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213550:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761233963.213554:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761233963.213555:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761233963.213571:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761233963.213575:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:3.0:1761233963.213581:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:3.0:1761233963.213586:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:3.0:1761233963.213590:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213593:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233963.213596:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213600:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761233963.213604:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:3.0:1761233963.213612:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761233963.213615:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233963.213620:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233963.213626:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761233963.213631:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213639:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761233963.213643:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761233963.213650:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213655:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761233963.213663:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000db341c88. 00000020:00000040:3.0:1761233963.213671:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:3.0:1761233963.213678:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761233963.213680:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213685:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233963.213687:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233963.213691:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233963.213700:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000009baf651a. 00000100:00000001:3.0:1761233963.213733:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233963.213740:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db341c88 x1846784245223808/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233963.213752:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233963.213760:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233963.213768:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000db341c88] to pc [ptlrpcd_rcv+-1] req@00000000db341c88 x1846784245223808/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233963.213785:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233963.213791:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:3.0:1761233963.213798:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6442) 00000800:00000200:2.1:1761233963.213804:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233963.213810:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761233963.214262:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233963.214265:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761233963.214651:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233963.214653:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000020:00000001:1.0:1761233964.109074:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233964.109079:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233964.109082:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233965.133076:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233965.133079:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233965.133082:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233966.157100:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233966.157104:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233966.157107:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233967.181072:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233967.181076:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233967.181079:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233968.205131:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233968.205134:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233968.205137:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233968.333103:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233968.333115:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233968.333117:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233968.333120:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233968.333123:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233968.333128:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233968.333135:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6437 00000100:00080000:0.0:1761233968.333138:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233968.333141:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333143:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233968.333145:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333147:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233968.333150:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233968.333152:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233968.333154:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233968.333157:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233968.333159:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233968.333160:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233968.333162:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333165:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233968.333166:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333167:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233968.333170:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233968.333173:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333174:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233968.333177:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233968.333179:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233968.333181:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333183:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233968.333185:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233968.333195:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ad5ae63. 00000020:00000040:0.0:1761233968.333198:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233968.333202:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233968.333203:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333206:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233968.333209:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233968.333213:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233968.333221:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233968.333228:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233968.333236:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ad5ae63 x1846784245223872/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233968.333243:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233968.333245:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233968.333249:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ad5ae63] to pc [ptlrpcd_rcv+-1] req@000000009ad5ae63 x1846784245223872/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233968.333267:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333269:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233968.333270:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233968.333273:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233968.333274:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233968.333275:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233968.333276:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233968.333278:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233968.333280:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6437 00000100:00080000:0.0:1761233968.333281:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233968.333283:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333284:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233968.333285:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333286:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233968.333287:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233968.333287:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233968.333288:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233968.333289:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233968.333290:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233968.333292:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233968.333294:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333295:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233968.333296:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333298:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233968.333299:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233968.333301:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333303:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233968.333304:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233968.333310:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233968.333312:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333313:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233968.333317:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233968.333320:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333322:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233968.333325:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000006edc786f. 00000020:00000040:0.0:1761233968.333331:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233968.333333:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233968.333335:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333338:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233968.333339:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233968.333340:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233968.333344:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233968.333348:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233968.333352:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006edc786f x1846784245223936/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233968.333362:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233968.333363:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233968.333367:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000006edc786f] to pc [ptlrpcd_rcv+-1] req@000000006edc786f x1846784245223936/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233968.333428:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333433:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233968.333436:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233968.333440:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233968.333441:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233968.333443:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233968.333445:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233968.333448:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233968.333452:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6437 00000100:00080000:0.0:1761233968.333455:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233968.333459:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333465:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233968.333470:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333473:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233968.333476:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233968.333477:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233968.333480:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233968.333485:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233968.333488:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233968.333493:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233968.333498:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333500:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233968.333501:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333502:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233968.333503:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233968.333506:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233968.333509:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233968.333512:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233968.333516:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233968.333525:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333528:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233968.333529:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233968.333531:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333537:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233968.333543:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761233968.333545:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233968.333550:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233968.333553:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333556:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233968.333556:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233968.333558:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233968.333561:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233968.333568:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233968.333576:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245224000/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233968.333584:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233968.333585:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233968.333594:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245224000/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233968.333657:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233968.333660:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233968.333666:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6447) 00000800:00000200:2.1:1761233968.333762:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233968.333765:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233968.334051:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233968.334052:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761233968.334203:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233968.334205:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000020:00000001:1.0:1761233969.229026:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233969.229028:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233969.229031:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233970.253157:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233970.253161:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233970.253165:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233971.277105:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233971.277111:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233971.277117:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233972.301034:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233972.301039:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233972.301043:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233973.325040:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233973.325044:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233973.325047:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233973.452961:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233973.452970:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233973.452972:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233973.452974:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233973.452975:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233973.452980:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233973.452985:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6442 00000100:00080000:0.0:1761233973.452987:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233973.452989:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.452991:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233973.452992:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.452994:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233973.452997:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233973.452998:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233973.453000:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233973.453003:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233973.453004:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233973.453006:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233973.453008:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453009:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233973.453010:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453011:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233973.453012:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233973.453015:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453016:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233973.453018:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233973.453021:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233973.453023:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453024:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233973.453025:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233973.453032:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761233973.453036:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233973.453039:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233973.453040:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453043:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233973.453045:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233973.453048:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233973.453054:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233973.453059:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233973.453064:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245224064/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233973.453071:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233973.453073:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233973.453075:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245224064/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233973.453093:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453094:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233973.453095:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233973.453097:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233973.453098:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233973.453099:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233973.453100:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233973.453102:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233973.453103:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6442 00000100:00080000:0.0:1761233973.453104:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233973.453105:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453106:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233973.453107:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453108:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233973.453109:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233973.453110:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233973.453111:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233973.453112:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233973.453113:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233973.453115:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233973.453116:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453118:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233973.453119:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453120:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233973.453121:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233973.453122:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453123:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233973.453124:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233973.453129:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233973.453130:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453132:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233973.453134:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233973.453136:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453138:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233973.453140:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761233973.453141:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233973.453142:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233973.453143:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453145:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233973.453147:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233973.453149:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233973.453154:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233973.453159:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233973.453166:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245224128/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233973.453176:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233973.453178:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233973.453181:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245224128/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233973.453195:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453198:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233973.453201:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233973.453204:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233973.453205:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233973.453206:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233973.453207:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233973.453211:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233973.453215:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6442 00000100:00080000:0.0:1761233973.453217:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233973.453221:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453223:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233973.453224:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453225:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233973.453228:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233973.453229:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233973.453230:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233973.453232:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233973.453237:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233973.453240:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233973.453246:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453250:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233973.453253:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453255:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233973.453259:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233973.453262:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233973.453267:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233973.453271:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233973.453274:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233973.453277:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453278:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233973.453281:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233973.453284:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453286:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233973.453289:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761233973.453290:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233973.453292:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233973.453293:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453294:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233973.453295:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233973.453297:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233973.453301:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233973.453304:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233973.453312:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245224192/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233973.453321:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233973.453323:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233973.453329:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245224192/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233973.453347:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233973.453349:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233973.453354:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6452) 00000800:00000200:2.1:1761233973.453368:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233973.453370:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761233973.453619:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233973.453621:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761233973.453758:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233973.453759:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000020:00000001:1.0:1761233974.349042:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233974.349046:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233974.349049:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233975.373070:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233975.373074:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233975.373077:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233976.397022:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233976.397025:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233976.397027:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233977.421111:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233977.421114:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233977.421116:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233978.445164:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233978.445171:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233978.445177:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233978.573045:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233978.573057:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233978.573059:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233978.573061:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233978.573064:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233978.573086:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233978.573094:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6447 00000100:00080000:0.0:1761233978.573098:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233978.573102:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573105:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233978.573108:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573111:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233978.573116:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233978.573119:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233978.573122:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233978.573126:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233978.573128:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233978.573131:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233978.573134:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573136:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233978.573138:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573140:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233978.573143:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233978.573147:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573150:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233978.573154:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233978.573157:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233978.573160:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573163:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233978.573165:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233978.573177:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761233978.573181:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233978.573186:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233978.573187:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573191:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233978.573194:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233978.573198:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233978.573207:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233978.573215:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233978.573224:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245224256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233978.573250:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233978.573252:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233978.573255:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245224256/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233978.573285:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573287:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233978.573288:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233978.573291:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233978.573292:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233978.573293:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233978.573294:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233978.573296:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233978.573298:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6447 00000100:00080000:0.0:1761233978.573300:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233978.573302:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573303:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233978.573305:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573306:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233978.573308:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233978.573309:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233978.573310:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233978.573311:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233978.573312:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233978.573314:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233978.573316:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573317:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233978.573318:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573319:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233978.573321:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233978.573322:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573324:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233978.573325:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233978.573330:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233978.573332:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573334:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233978.573337:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233978.573340:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573341:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233978.573344:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761233978.573346:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233978.573348:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233978.573349:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573351:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233978.573352:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233978.573355:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233978.573360:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233978.573365:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233978.573372:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245224320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233978.573406:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233978.573409:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233978.573416:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245224320/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233978.573445:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573447:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233978.573448:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233978.573450:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233978.573451:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233978.573452:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233978.573453:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233978.573456:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233978.573459:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6447 00000100:00080000:0.0:1761233978.573463:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233978.573468:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573473:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233978.573477:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573481:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233978.573485:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233978.573487:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233978.573489:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233978.573492:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233978.573494:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233978.573498:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233978.573501:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573503:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233978.573505:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573510:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233978.573514:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233978.573517:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233978.573519:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233978.573522:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233978.573526:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233978.573528:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573532:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233978.573534:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233978.573536:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573537:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233978.573544:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761233978.573549:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233978.573553:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233978.573555:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573557:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233978.573559:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233978.573561:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233978.573568:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233978.573572:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233978.573581:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245224384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233978.573610:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233978.573612:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233978.573617:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245224384/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233978.573623:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233978.573624:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233978.573628:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6457) 00000800:00000200:2.1:1761233978.573683:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233978.573685:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761233978.573987:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233978.573989:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761233978.574165:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233978.574167:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000020:00000001:1.0:1761233979.469135:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233979.469140:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233979.469145:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233980.493077:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233980.493081:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233980.493086:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233981.517003:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233981.517006:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233981.517008:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233982.541233:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233982.541239:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233982.541245:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233983.565123:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233983.565126:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233983.565129:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233983.692996:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233983.693009:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233983.693012:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233983.693015:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233983.693018:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233983.693024:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233983.693031:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6452 00000100:00080000:0.0:1761233983.693036:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233983.693040:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693043:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233983.693047:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693051:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233983.693056:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233983.693058:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233983.693062:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233983.693066:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233983.693069:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233983.693072:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233983.693076:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693079:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233983.693082:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693085:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233983.693089:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233983.693094:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693098:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233983.693102:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233983.693105:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233983.693109:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693110:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233983.693112:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233983.693119:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761233983.693122:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233983.693126:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233983.693127:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693130:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233983.693132:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233983.693136:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233983.693142:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233983.693148:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233983.693155:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245224448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233983.693163:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233983.693165:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233983.693168:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245224448/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233983.693187:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693189:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233983.693190:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233983.693193:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233983.693194:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233983.693195:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233983.693196:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233983.693198:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233983.693199:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6452 00000100:00080000:0.0:1761233983.693201:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233983.693203:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693205:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233983.693206:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693207:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233983.693209:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233983.693210:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233983.693212:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233983.693213:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233983.693215:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233983.693216:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233983.693219:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693221:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233983.693223:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693224:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233983.693226:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233983.693229:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693231:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233983.693233:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233983.693238:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233983.693242:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693245:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233983.693249:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233983.693254:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693257:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233983.693262:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761233983.693265:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233983.693270:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233983.693273:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693278:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233983.693280:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233983.693285:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233983.693293:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233983.693298:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233983.693306:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245224512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233983.693316:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233983.693318:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233983.693358:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245224512/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233983.693368:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693374:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233983.693377:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233983.693383:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233983.693385:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233983.693388:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233983.693390:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233983.693395:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233983.693399:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6452 00000100:00080000:0.0:1761233983.693403:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233983.693408:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693411:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233983.693414:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693417:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233983.693421:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233983.693422:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233983.693424:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233983.693428:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233983.693432:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233983.693436:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233983.693440:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693444:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233983.693448:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693452:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233983.693453:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233983.693455:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233983.693457:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233983.693458:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233983.693461:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233983.693465:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693468:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233983.693469:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233983.693473:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693476:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233983.693482:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761233983.693487:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233983.693492:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233983.693494:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693498:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233983.693499:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233983.693501:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233983.693507:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233983.693511:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233983.693518:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245224576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233983.693528:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233983.693530:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233983.693536:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245224576/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233983.693550:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233983.693554:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:2.1:1761233983.693554:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233983.693558:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000100:00000040:0.0:1761233983.693559:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6462) 00000800:00000200:2.1:1761233983.693815:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233983.693816:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:2.1:1761233983.694029:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233983.694030:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000020:00000001:1.0:1761233984.589040:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233984.589044:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233984.589046:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233985.613056:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233985.613059:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233985.613062:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233986.637032:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233986.637035:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233986.637037:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233987.661019:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233987.661022:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233987.661025:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233988.685062:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233988.685065:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233988.685068:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233988.812974:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233988.812984:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233988.812985:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233988.812988:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233988.812990:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233988.812994:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233988.812999:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6457 00000100:00080000:0.0:1761233988.813002:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233988.813004:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813006:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233988.813007:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813009:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233988.813012:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233988.813013:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233988.813015:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233988.813017:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233988.813018:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233988.813020:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233988.813021:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813023:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233988.813024:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813025:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233988.813027:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233988.813029:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813030:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233988.813032:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233988.813035:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233988.813036:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813037:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233988.813039:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233988.813047:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761233988.813049:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233988.813053:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233988.813054:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813056:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233988.813058:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233988.813061:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233988.813068:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761233988.813072:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233988.813077:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245224640/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233988.813084:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233988.813085:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233988.813088:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245224640/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233988.813106:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813107:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233988.813109:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233988.813111:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233988.813111:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233988.813112:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233988.813113:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233988.813115:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233988.813116:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6457 00000100:00080000:0.0:1761233988.813117:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233988.813118:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813120:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233988.813120:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813121:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233988.813122:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233988.813123:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233988.813124:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233988.813125:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233988.813126:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233988.813127:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233988.813128:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813130:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233988.813131:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813132:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233988.813133:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233988.813134:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813135:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233988.813136:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233988.813140:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233988.813141:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813142:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233988.813144:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233988.813146:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813147:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233988.813149:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761233988.813150:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233988.813151:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233988.813152:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813152:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233988.813153:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233988.813154:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233988.813156:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761233988.813158:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233988.813160:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245224704/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233988.813164:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233988.813165:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233988.813167:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245224704/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233988.813171:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813172:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233988.813173:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233988.813176:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233988.813177:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233988.813179:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233988.813179:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233988.813181:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233988.813183:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6457 00000100:00080000:0.0:1761233988.813186:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233988.813188:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813192:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233988.813194:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813196:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233988.813198:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233988.813199:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233988.813201:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233988.813203:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233988.813205:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233988.813208:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233988.813214:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813217:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233988.813220:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813223:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233988.813243:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233988.813248:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233988.813250:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233988.813253:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233988.813257:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233988.813260:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813262:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233988.813263:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233988.813266:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813270:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233988.813275:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233988.813277:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233988.813281:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233988.813282:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813283:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233988.813284:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233988.813286:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233988.813292:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233988.813297:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233988.813302:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245224768/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233988.813324:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233988.813326:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233988.813332:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245224768/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233988.813344:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233988.813347:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233988.813351:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6467) 00000800:00000200:2.1:1761233988.813403:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233988.813405:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761233988.813588:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233988.813589:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000800:00000200:2.1:1761233988.813793:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233988.813794:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 28 secs later) 00000020:00000001:1.0:1761233989.709062:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233989.709066:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233989.709069:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233990.733133:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233990.733137:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233990.733140:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233991.757035:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233991.757038:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233991.757042:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233992.781057:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233992.781059:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233992.781062:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233993.805034:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233993.805036:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233993.805038:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761233993.932940:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233993.932955:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233993.932957:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761233993.932960:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233993.932963:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233993.932968:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233993.932982:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6462 00000100:00080000:0.0:1761233993.932985:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233993.932988:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.932990:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233993.932995:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.932997:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233993.933001:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233993.933003:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761233993.933006:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233993.933009:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761233993.933011:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761233993.933013:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761233993.933015:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933016:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233993.933017:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933018:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233993.933020:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761233993.933023:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933024:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233993.933026:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233993.933029:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233993.933031:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933033:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761233993.933035:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233993.933046:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002ca01b78. 00000020:00000040:0.0:1761233993.933050:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761233993.933054:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233993.933055:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933059:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233993.933061:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233993.933066:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233993.933074:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761233993.933081:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233993.933087:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ca01b78 x1846784245224832/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233993.933095:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233993.933097:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233993.933100:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002ca01b78] to pc [ptlrpcd_rcv+-1] req@000000002ca01b78 x1846784245224832/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233993.933118:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933119:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233993.933121:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233993.933124:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233993.933125:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761233993.933126:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233993.933126:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233993.933128:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233993.933142:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6462 00000100:00080000:0.0:1761233993.933144:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233993.933145:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933147:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233993.933148:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933149:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233993.933150:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233993.933151:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761233993.933152:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233993.933153:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761233993.933154:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761233993.933156:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761233993.933158:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933160:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233993.933161:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933162:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233993.933163:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761233993.933164:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933166:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233993.933167:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233993.933173:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233993.933174:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933175:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233993.933179:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233993.933182:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933184:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233993.933187:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000095969832. 00000020:00000040:0.0:1761233993.933188:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761233993.933190:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233993.933192:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933194:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233993.933195:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233993.933197:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233993.933201:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761233993.933205:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233993.933213:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000095969832 x1846784245224896/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233993.933221:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233993.933223:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233993.933230:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000095969832] to pc [ptlrpcd_rcv+-1] req@0000000095969832 x1846784245224896/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233993.933279:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933282:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761233993.933285:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761233993.933294:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761233993.933295:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761233993.933298:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761233993.933300:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761233993.933303:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761233993.933307:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6462 00000100:00080000:0.0:1761233993.933310:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761233993.933315:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933318:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233993.933321:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933326:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761233993.933332:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761233993.933333:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761233993.933335:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761233993.933337:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761233993.933338:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761233993.933340:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761233993.933348:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933352:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761233993.933355:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933356:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761233993.933358:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761233993.933360:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761233993.933361:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761233993.933362:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761233993.933364:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761233993.933365:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933370:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761233993.933373:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761233993.933377:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933378:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761233993.933383:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761233993.933386:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761233993.933389:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761233993.933390:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933392:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761233993.933392:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761233993.933395:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761233993.933400:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761233993.933403:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761233993.933411:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245224960/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233993.933423:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761233993.933424:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761233993.933430:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245224960/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761233993.933474:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761233993.933485:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761233993.933490:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6473) 00000800:00000200:2.1:1761233993.933583:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233993.933586:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761233993.933952:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233993.933954:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761233993.934200:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233993.934202:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000020:00000001:1.0:1761233994.829081:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233994.829086:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233994.829092:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233995.853055:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233995.853058:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233995.853061:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233996.877053:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233996.877056:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233996.877058:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233997.901108:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233997.901112:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233997.901116:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761233998.925170:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233998.925174:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233998.925177:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761233999.053068:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761233999.053080:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761233999.053082:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761233999.053085:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761233999.053088:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761233999.053094:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761233999.053100:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6468 00000100:00080000:3.0:1761233999.053103:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761233999.053106:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053108:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233999.053110:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053112:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761233999.053115:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761233999.053117:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761233999.053119:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761233999.053124:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761233999.053125:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761233999.053127:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761233999.053129:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053130:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233999.053131:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053132:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761233999.053134:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761233999.053137:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053139:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233999.053141:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233999.053143:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761233999.053145:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053147:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761233999.053149:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761233999.053159:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f61de4c8. 00000020:00000040:3.0:1761233999.053162:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761233999.053166:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761233999.053167:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053170:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233999.053173:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233999.053177:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233999.053185:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000df51b12a. 00000100:00000001:3.0:1761233999.053192:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233999.053199:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f61de4c8 x1846784245225024/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233999.053207:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233999.053209:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233999.053219:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f61de4c8] to pc [ptlrpcd_rcv+-1] req@00000000f61de4c8 x1846784245225024/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233999.053236:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053238:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761233999.053240:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761233999.053243:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761233999.053244:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761233999.053245:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761233999.053246:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761233999.053247:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761233999.053249:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6468 00000100:00080000:3.0:1761233999.053250:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761233999.053251:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053252:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233999.053253:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053254:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761233999.053255:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761233999.053256:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761233999.053257:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761233999.053258:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761233999.053259:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761233999.053260:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761233999.053262:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053263:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233999.053264:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053265:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761233999.053267:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761233999.053268:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053269:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233999.053270:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233999.053275:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761233999.053277:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053278:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761233999.053282:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761233999.053285:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053287:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761233999.053290:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002431276f. 00000020:00000040:3.0:1761233999.053291:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761233999.053293:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761233999.053294:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053295:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233999.053297:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233999.053299:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233999.053302:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000071d5cb10. 00000100:00000001:3.0:1761233999.053307:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233999.053311:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002431276f x1846784245225088/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233999.053316:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233999.053317:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233999.053322:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002431276f] to pc [ptlrpcd_rcv+-1] req@000000002431276f x1846784245225088/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233999.053334:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053337:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761233999.053339:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761233999.053342:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761233999.053344:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761233999.053345:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761233999.053348:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761233999.053350:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761233999.053354:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6468 00000100:00080000:3.0:1761233999.053358:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761233999.053364:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053366:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233999.053367:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053368:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761233999.053371:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761233999.053373:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761233999.053376:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761233999.053378:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:3.0:1761233999.053380:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:3.0:1761233999.053381:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:3.0:1761233999.053383:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053388:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761233999.053391:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053392:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761233999.053393:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:3.0:1761233999.053395:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761233999.053399:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761233999.053400:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761233999.053402:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761233999.053404:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053405:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761233999.053408:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761233999.053410:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053411:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761233999.053414:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000108ab0a5. 00000020:00000040:3.0:1761233999.053418:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:3.0:1761233999.053421:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761233999.053423:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053424:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761233999.053427:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761233999.053429:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761233999.053433:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000001c1b0e94. 00000100:00000001:3.0:1761233999.053436:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761233999.053444:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000108ab0a5 x1846784245225152/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233999.053451:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761233999.053453:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761233999.053457:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000108ab0a5] to pc [ptlrpcd_rcv+-1] req@00000000108ab0a5 x1846784245225152/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761233999.053465:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761233999.053467:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:3.0:1761233999.053470:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6478) 00000800:00000200:2.1:1761233999.053539:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233999.053541:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761233999.053779:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233999.053781:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761233999.053952:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761233999.053954:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000020:00000001:1.0:1761233999.949111:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761233999.949116:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761233999.949121:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761234000.973120:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234000.973123:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234000.973125:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761234001.997111:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234001.997117:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234001.997122:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761234003.021071:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234003.021074:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234003.021078:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761234004.045019:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234004.045023:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234004.045026:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234004.173080:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234004.173092:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234004.173094:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234004.173097:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234004.173100:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234004.173104:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234004.173112:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6473 00000100:00080000:1.0:1761234004.173116:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234004.173118:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173120:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234004.173122:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173124:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234004.173127:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234004.173129:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234004.173131:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234004.173134:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234004.173136:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234004.173138:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234004.173140:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173141:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234004.173142:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173143:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234004.173145:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234004.173148:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173149:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234004.173152:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234004.173155:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234004.173157:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173159:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234004.173160:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234004.173171:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761234004.173175:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234004.173179:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234004.173180:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173183:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234004.173185:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234004.173189:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234004.173197:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761234004.173203:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234004.173210:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245225216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234004.173219:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234004.173221:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234004.173224:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245225216/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234004.173247:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173249:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234004.173251:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234004.173255:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234004.173256:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234004.173257:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234004.173258:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234004.173261:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234004.173263:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6473 00000100:00080000:1.0:1761234004.173265:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234004.173268:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173270:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234004.173272:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173275:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234004.173277:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234004.173278:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234004.173280:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234004.173282:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234004.173284:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234004.173286:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234004.173290:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173293:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234004.173297:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173302:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234004.173307:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234004.173314:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173320:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234004.173323:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234004.173330:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234004.173337:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173342:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234004.173348:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234004.173353:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173357:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234004.173363:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000020:00000040:1.0:1761234004.173367:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234004.173373:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234004.173375:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173379:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234004.173382:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234004.173383:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234004.173389:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234004.173395:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234004.173404:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245225280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234004.173435:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234004.173438:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234004.173446:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245225280/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234004.173463:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173466:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234004.173467:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234004.173471:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234004.173474:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234004.173477:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234004.173478:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234004.173483:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234004.173490:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6473 00000100:00080000:1.0:1761234004.173495:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234004.173503:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173507:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234004.173513:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173518:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234004.173523:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234004.173525:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234004.173528:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234004.173532:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234004.173536:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234004.173541:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234004.173549:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000800:00000200:2.1:1761234004.173552:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:1.0:1761234004.173555:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1761234004.173557:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000100:00000040:1.0:1761234004.173561:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173567:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234004.173573:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234004.173580:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234004.173587:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234004.173590:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234004.173595:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234004.173599:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173602:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234004.173605:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234004.173610:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173613:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234004.173618:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761234004.173621:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234004.173626:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234004.173627:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173631:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234004.173632:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234004.173634:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234004.173639:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761234004.173643:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234004.173650:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245225344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234004.173662:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234004.173664:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234004.173671:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245225344/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234004.173682:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234004.173684:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234004.173688:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6483) 00000800:00000200:2.1:1761234004.173898:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234004.173900:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761234004.174163:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234004.174166:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000020:00000001:1.0:1761234005.069080:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234005.069084:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234005.069088:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761234006.093019:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234006.093026:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234006.093031:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761234007.117131:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234007.117135:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234007.117137:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761234008.141118:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234008.141123:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234008.141128:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1761234009.165155:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234009.165159:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234009.165163:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234009.292966:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234009.292975:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234009.292977:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234009.292979:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234009.292981:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234009.292987:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234009.292992:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6478 00000100:00080000:1.0:1761234009.292995:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234009.292997:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.292999:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234009.293000:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293002:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234009.293004:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234009.293006:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234009.293008:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234009.293010:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234009.293011:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234009.293013:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234009.293014:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293015:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234009.293016:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293018:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234009.293019:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234009.293022:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293023:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234009.293026:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234009.293027:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234009.293029:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293030:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234009.293032:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234009.293040:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761234009.293042:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234009.293045:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234009.293046:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293048:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234009.293050:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234009.293053:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234009.293060:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234009.293064:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234009.293069:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245225408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234009.293076:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234009.293078:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234009.293081:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245225408/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234009.293097:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293100:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234009.293101:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234009.293104:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234009.293105:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234009.293106:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234009.293107:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234009.293110:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234009.293112:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6478 00000100:00080000:1.0:1761234009.293114:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234009.293116:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293118:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234009.293120:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293121:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234009.293123:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234009.293124:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234009.293125:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234009.293127:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234009.293129:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234009.293130:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234009.293133:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293135:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234009.293137:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293138:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234009.293140:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234009.293142:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293144:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234009.293145:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234009.293151:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234009.293153:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293155:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234009.293157:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234009.293160:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293162:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234009.293165:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761234009.293167:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234009.293170:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234009.293171:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293172:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234009.293173:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234009.293175:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234009.293178:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761234009.293181:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234009.293184:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245225472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234009.293190:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234009.293191:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234009.293193:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245225472/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234009.293200:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293202:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234009.293203:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234009.293206:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234009.293207:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234009.293209:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234009.293210:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234009.293212:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234009.293214:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6478 00000100:00080000:1.0:1761234009.293216:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234009.293218:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293220:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234009.293222:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293223:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234009.293225:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234009.293226:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234009.293228:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234009.293229:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234009.293231:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234009.293233:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234009.293235:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293237:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234009.293238:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293240:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234009.293241:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234009.293243:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234009.293245:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234009.293246:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234009.293248:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234009.293250:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293251:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234009.293252:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234009.293254:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293256:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234009.293258:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761234009.293259:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234009.293262:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234009.293263:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293264:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234009.293265:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234009.293266:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234009.293269:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761234009.293272:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234009.293274:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245225536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234009.293280:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234009.293281:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234009.293284:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245225536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234009.293290:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234009.293292:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234009.293295:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6488) 00000800:00000200:2.1:1761234009.293317:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234009.293320:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761234009.293591:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234009.293592:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761234009.293733:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234009.293734:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 02000000:00000001:2.0:1761234009.293890:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234009.293891:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007d6a5077. 00000100:00000001:2.0:1761234009.293893:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234009.293893:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234009.293894:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234009.293895:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000643267a x1846784245225536/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234034 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234009.293899:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234009.293899:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234009.293900:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000dd1dcb0e. 02000000:00000001:2.0:1761234009.293902:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234009.293902:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234009.293903:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234009.293904:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234009.293904:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f654abd8. 02000000:00000001:2.0:1761234009.293906:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234009.293907:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234009.293907:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000643267a. 00000100:00000001:2.0:1761234009.293909:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234009.293909:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234009.293910:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234009.293913:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234009.293914:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234009.293915:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234009.293916:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234009.804956:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234009.804961:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234009.804962:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234009.804966:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234009.804971:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234009.804975:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234009.804977:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234009.804980:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234009.804982:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234010.188992:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234010.189000:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234010.189009:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234010.189011:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234010.189015:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234010.189016:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234010.189118:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234010.189124:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234010.189129:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234010.828981:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234010.828988:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234010.828989:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234010.828994:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234010.828999:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234010.829003:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234010.829005:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234010.829008:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234010.829010:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234011.212956:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234011.212961:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234011.212966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234011.212967:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234011.212970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234011.212971:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234011.213035:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234011.213037:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234011.213039:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234011.852948:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234011.852954:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234011.852955:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234011.852958:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234011.852962:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234011.852965:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234011.852967:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234011.852968:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234011.852970:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234012.237027:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234012.237032:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234012.237037:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234012.237038:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234012.237041:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234012.237043:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234012.237107:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234012.237110:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234012.237113:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234012.877054:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234012.877060:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234012.877061:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234012.877066:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234012.877070:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234012.877074:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234012.877076:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234012.877079:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234012.877081:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234013.260959:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234013.260963:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234013.260969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234013.260970:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234013.260972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234013.260973:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234013.261047:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234013.261050:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234013.261053:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234013.900958:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234013.900965:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234013.900967:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234013.900971:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234013.900975:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234013.900979:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234013.900981:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234013.900983:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234013.900986:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234014.284939:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234014.284943:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234014.284947:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234014.284948:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234014.284950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234014.284951:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234014.285006:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234014.285009:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234014.285011:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234014.412952:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234014.412961:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234014.412963:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234014.412966:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234014.412968:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234014.412972:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234014.412977:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6483 00000100:00080000:1.0:1761234014.412979:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234014.412981:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.412983:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234014.412985:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.412986:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234014.412989:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234014.412990:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234014.412992:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234014.412994:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234014.412996:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234014.412997:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234014.412999:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413000:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234014.413001:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413002:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234014.413004:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234014.413007:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413008:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234014.413010:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234014.413012:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234014.413014:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413015:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234014.413017:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234014.413024:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761234014.413027:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234014.413030:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234014.413031:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413033:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234014.413035:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234014.413038:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234014.413044:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761234014.413048:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234014.413053:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234014.413060:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234014.413062:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234014.413064:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234014.413080:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413083:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234014.413084:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234014.413087:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234014.413088:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234014.413089:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234014.413090:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234014.413093:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234014.413095:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6483 00000100:00080000:1.0:1761234014.413097:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234014.413099:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413101:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234014.413107:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413109:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234014.413111:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234014.413112:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234014.413113:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234014.413115:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234014.413117:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234014.413118:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234014.413121:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413123:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234014.413124:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413127:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234014.413128:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234014.413130:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234014.413130:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234014.413132:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234014.413134:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1761234014.413134:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234014.413136:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234014.413139:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1761234014.413140:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:1.0:1761234014.413141:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413142:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234014.413145:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1761234014.413147:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1761234014.413148:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413148:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413150:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234014.413152:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:1.0:1761234014.413153:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 02000000:00000001:2.0:1761234014.413154:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1761234014.413155:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00100000:2.0:1761234014.413156:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245225600:192.168.201.124@tcp:250: 02000000:00000001:1.0:1761234014.413157:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234014.413158:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413160:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234014.413160:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761234014.413161:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234014.413162:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234014.413164:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:1.0:1761234014.413165:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 02000000:00000001:2.0:1761234014.413166:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413168:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234014.413169:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1761234014.413171:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:2.0:1761234014.413173:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234014.413174:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413178:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234014.413179:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234014.413179:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000a1db4867. 00000100:00000040:1.0:1761234014.413181:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:2.0:1761234014.413185:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000053c991f. 00000100:00000001:1.0:1761234014.413187:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413189:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234014.413190:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234014.413193:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234014.413195:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234014.413196:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234014.413197:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234014.413200:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00000200:2.0:1761234014.413200:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245225600, portal 25 00000100:00080000:1.0:1761234014.413202:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6483 00000100:00000001:2.0:1761234014.413202:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1761234014.413203:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1761234014.413204:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039795456 : -104717669756160 : ffffa0c2847e1300) 00000100:00000040:1.0:1761234014.413206:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413208:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234014.413208:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234039 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:1.0:1761234014.413209:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413211:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234014.413213:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234014.413214:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234014.413215:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234014.413217:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234014.413218:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:2.0:1761234014.413218:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000001:1.0:1761234014.413220:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234014.413222:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000040:1.0:1761234014.413223:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413224:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1761234014.413224:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000001d64cf84. 00000100:00000040:1.0:1761234014.413226:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000200:2.0:1761234014.413227:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245225600, offset 0 00000100:00000001:1.0:1761234014.413228:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234014.413229:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234014.413231:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234014.413234:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234014.413234:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:1.0:1761234014.413235:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234014.413237:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234014.413238:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413240:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234014.413241:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234014.413243:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413244:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234014.413246:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000400:00000200:2.0:1761234014.413247:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000020:00000040:1.0:1761234014.413249:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234014.413251:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234014.413252:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413253:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234014.413254:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1761234014.413254:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 02000000:00000001:1.0:1761234014.413255:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234014.413258:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000400:00000200:2.0:1761234014.413258:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1761234014.413260:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234014.413261:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00100000:1.0:1761234014.413263:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234014.413264:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11742:11742:256:4294967295] 192.168.201.124@tcp LPNI seq info [11742:11742:8:4294967295] 00000100:00000001:1.0:1761234014.413269:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234014.413270:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234014.413273:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234014.413275:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:1.0:1761234014.413280:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234014.413281:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:2.0:1761234014.413281:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000040:1.0:1761234014.413285:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6493) 00000800:00000010:2.0:1761234014.413286:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009b62b3c1 (tot 1676530). 00000800:00000200:2.0:1761234014.413292:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234014.413295:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761234014.413301:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234014.413304:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1761234014.413308:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234014.413310:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009b62b3c1 (tot 1676282). 00000400:00000200:2.0:1761234014.413316:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234014.413321:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234014.413326:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000001d64cf84 00000400:00000010:2.0:1761234014.413328:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000001d64cf84. 00000100:00000001:2.0:1761234014.413333:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234014.413335:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234039 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413342:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234014.413344:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234039 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413348:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413349:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234014.413354:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413355:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234014.413357:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413359:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234014.413361:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234014/real 1761234014] req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234014.413367:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234014.413368:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234014.413370:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000a1db4867. 00000400:00000200:2.0:1761234014.413371:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000053c991f 00000400:00000010:2.0:1761234014.413372:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000053c991f. 00000100:00000001:2.0:1761234014.413374:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234014.413375:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234014.413380:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413383:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234014.413384:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413385:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234014.413386:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234014.413388:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413392:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234014.413394:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234014.413398:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234014.413400:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413401:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234014.413402:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413403:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234014.413404:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234014.413406:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234014.413407:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234014.413408:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234014.413410:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234014.413415:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245225600:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234014.413417:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413419:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234014.413420:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fd54b549 x1846784245225600/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234014.413424:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234014.413426:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234014.413428:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234014.413430:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234014.413431:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234014.413431:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234014.413433:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234014.413434:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006d8d01a9. 02000000:00000001:2.0:1761234014.413436:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234014.413437:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234014.413438:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fd54b549. 00000100:00000001:2.0:1761234014.413441:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234014.413441:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413442:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413448:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234014.413450:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413451:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234014.413452:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234014.413453:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234014.413454:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413458:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234014.413459:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234014.413460:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234014.413461:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234014.413463:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245225664:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234014.413465:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234014.413466:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234014.413467:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234014.413468:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234014.413469:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234014.413470:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234014.413472:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000a1db4867. 00000400:00000010:2.0:1761234014.413474:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000053c991f. 00000100:00000200:2.0:1761234014.413477:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245225664, portal 10 00000100:00000001:2.0:1761234014.413478:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234014.413480:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039808832 : -104717669742784 : ffffa0c2847e4740) 00000100:00000040:2.0:1761234014.413484:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234039 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413488:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234014.413490:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234014.413491:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000001d64cf84. 00000100:00000200:2.0:1761234014.413493:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245225664, offset 0 00000400:00000200:2.0:1761234014.413495:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234014.413498:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234014.413501:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234014.413502:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234014.413504:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234014.413506:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11743:11743:256:4294967295] 192.168.201.124@tcp LPNI seq info [11743:11743:8:4294967295] 00000400:00000200:2.0:1761234014.413511:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234014.413514:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234014.413516:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009b62b3c1 (tot 1676530). 00000800:00000200:2.0:1761234014.413518:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234014.413520:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761234014.413522:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234014.413523:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1761234014.413526:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234014.413527:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009b62b3c1 (tot 1676282). 00000400:00000200:2.0:1761234014.413529:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234014.413531:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234014.413532:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000001d64cf84 00000400:00000010:2.0:1761234014.413533:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000001d64cf84. 00000100:00000001:2.0:1761234014.413534:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234014.413536:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234039 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413540:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234014.413542:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234039 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413546:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413547:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234014.413548:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413549:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234014.413550:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413551:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234014.413553:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234014/real 1761234014] req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234014.413557:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234014.413558:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234014.413559:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000a1db4867. 00000400:00000200:2.0:1761234014.413560:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000053c991f 00000400:00000010:2.0:1761234014.413561:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000053c991f. 00000100:00000001:2.0:1761234014.413562:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234014.413563:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234014.413567:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413570:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234014.413571:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413572:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234014.413573:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234014.413574:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413577:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234014.413579:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234014.413582:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234014.413583:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413584:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234014.413585:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413586:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234014.413587:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234014.413589:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234014.413590:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234014.413591:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234014.413593:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234014.413597:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245225664:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234014.413599:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234014.413600:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413603:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234014.413604:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234014.413605:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234014.413606:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234014.413607:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245225728:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234014.413609:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234014.413610:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234014.413611:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234014.413612:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234014.413614:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234014.413615:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234014.413616:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000a1db4867. 00000400:00000010:2.0:1761234014.413618:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000053c991f. 00000100:00000200:2.0:1761234014.413619:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245225728, portal 4 00000100:00000001:2.0:1761234014.413620:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234014.413621:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039812480 : -104717669739136 : ffffa0c2847e5580) 00000100:00000040:2.0:1761234014.413624:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234039 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413627:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234014.413629:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234014.413630:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000001d64cf84. 00000100:00000200:2.0:1761234014.413631:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245225728, offset 0 00000400:00000200:2.0:1761234014.413634:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234014.413636:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234014.413638:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234014.413640:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234014.413641:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234014.413642:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11744:11744:256:4294967295] 192.168.201.124@tcp LPNI seq info [11744:11744:8:4294967295] 00000400:00000200:2.0:1761234014.413647:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234014.413649:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234014.413651:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009b62b3c1 (tot 1676530). 00000800:00000200:2.0:1761234014.413653:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234014.413654:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761234014.413657:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234014.413658:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1761234014.413660:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234014.413661:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009b62b3c1 (tot 1676282). 00000400:00000200:2.0:1761234014.413663:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234014.413665:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234014.413666:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000001d64cf84 00000400:00000010:2.0:1761234014.413667:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000001d64cf84. 00000100:00000001:2.0:1761234014.413668:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234014.413669:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234039 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413673:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234014.413674:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234039 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413678:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413679:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234014.413680:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413681:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234014.413682:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413683:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234014.413685:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234014/real 1761234014] req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234014.413689:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234014.413690:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234014.413691:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000a1db4867. 00000400:00000200:2.0:1761234014.413691:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000053c991f 00000400:00000010:2.0:1761234014.413692:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000053c991f. 00000100:00000001:2.0:1761234014.413693:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234014.413694:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234014.413698:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413701:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234014.413701:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413702:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234014.413703:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234014.413704:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234014.413708:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234014.413709:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234014.413712:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234014.413713:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413714:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234014.413715:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413716:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234014.413717:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234014.413718:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234014.413720:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234014.413721:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234014.413723:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234014.413727:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245225728:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234014.413729:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413730:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234014.413731:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f33bf85c x1846784245225664/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234014.413734:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234014.413735:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234014.413736:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234014.413737:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234014.413737:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234014.413738:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234014.413739:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234014.413740:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:2.0:1761234014.413760:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234014.413760:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234014.413761:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f33bf85c. 00000100:00000001:2.0:1761234014.413763:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234014.413763:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413764:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234014.413766:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004567def3 x1846784245225728/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234039 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234014.413770:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234014.413771:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234014.413771:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234014.413773:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234014.413774:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234014.413774:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234014.413775:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234014.413776:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:2.0:1761234014.413777:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234014.413778:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234014.413779:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004567def3. 00000100:00000001:2.0:1761234014.413780:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234014.413781:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413782:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234014.413784:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234014.413785:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234014.413786:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234014.413787:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234014.924938:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234014.924942:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234014.924944:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234014.924948:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234014.924953:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234014.924955:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234014.924958:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234014.924960:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234014.924963:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234015.308967:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234015.308972:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234015.308978:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234015.308979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234015.308981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234015.308982:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234015.309043:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234015.309046:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234015.309049:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234015.948938:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234015.948944:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234015.948946:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234015.948951:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234015.948957:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234015.948960:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234015.948963:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234015.948965:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234015.948967:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234016.332999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234016.333005:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234016.333011:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234016.333012:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234016.333016:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234016.333017:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234016.333112:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234016.333115:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234016.333119:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234016.972989:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234016.972995:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234016.972996:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234016.973001:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234016.973005:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234016.973008:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234016.973010:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234016.973013:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234016.973015:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234017.356957:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234017.356963:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234017.356970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234017.356971:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234017.356974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234017.356975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234017.357055:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234017.357060:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234017.357064:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234017.996949:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234017.996972:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234017.996974:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234017.996979:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234017.996985:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234017.996990:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234017.996992:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234017.996995:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234017.996997:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234018.380969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234018.380974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234018.380980:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234018.380981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234018.380984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234018.380984:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234018.381054:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234018.381058:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234018.381062:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234019.020975:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234019.020980:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234019.020981:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234019.020985:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234019.020989:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234019.020992:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234019.020994:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234019.020995:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234019.020997:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404946:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234019.404952:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234019.404955:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234019.404956:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404961:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234019.404962:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404965:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404966:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234019.404970:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234019.404971:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234019.404974:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234019.404976:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234019.404977:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404978:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234019.404978:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404979:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404980:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234019.404984:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234019.404984:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234019.404986:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234019.404986:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234019.404987:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404988:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234019.404988:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404989:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234019.404990:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234019.405065:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234019.405070:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234019.405074:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234019.533036:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234019.533049:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234019.533050:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234019.533053:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234019.533056:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234019.533062:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234019.533069:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6488 00000100:00080000:3.0:1761234019.533072:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234019.533074:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533077:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234019.533078:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533080:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234019.533083:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234019.533085:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234019.533087:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234019.533097:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234019.533100:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234019.533102:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234019.533103:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533105:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234019.533106:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533107:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234019.533109:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234019.533112:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533113:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234019.533117:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234019.533120:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234019.533123:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533125:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234019.533127:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234019.533142:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000108ab0a5. 00000020:00000040:3.0:1761234019.533145:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234019.533150:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234019.533151:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533154:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234019.533157:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234019.533177:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234019.533190:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000001c1b0e94. 00000100:00000001:3.0:1761234019.533198:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234019.533207:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234019.533217:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234019.533219:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234019.533223:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000108ab0a5] to pc [ptlrpcd_rcv+-1] req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234019.533248:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533250:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234019.533252:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234019.533256:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234019.533257:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234019.533258:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234019.533259:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234019.533262:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234019.533264:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6488 00000100:00080000:3.0:1761234019.533266:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234019.533268:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533269:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234019.533270:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533273:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234019.533275:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234019.533276:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234019.533277:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234019.533279:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234019.533281:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234019.533282:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234019.533284:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533285:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234019.533287:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533288:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234019.533289:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234019.533291:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533293:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234019.533294:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234019.533300:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234019.533302:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533303:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234019.533308:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234019.533312:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533314:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234019.533318:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002431276f. 00000020:00000040:3.0:1761234019.533320:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761234019.533322:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234019.533324:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533326:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234019.533326:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234019.533328:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234019.533332:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000071d5cb10. 00000100:00000001:3.0:1761234019.533335:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234019.533338:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234019.533345:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234019.533351:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234019.533352:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234019.533356:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.533357:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234019.533366:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002431276f] to pc [ptlrpcd_rcv+-1] req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234019.533374:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234019.533406:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.533408:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234019.533410:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000001:2.0:1761234019.533412:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1761234019.533413:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1761234019.533417:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1761234019.533419:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234019.533421:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1761234019.533422:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533424:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234019.533427:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1761234019.533431:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245225792:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234019.533433:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234019.533441:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6488 00000100:00000001:2.0:1761234019.533442:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1761234019.533445:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1761234019.533447:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234019.533450:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234019.533451:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533457:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1761234019.533458:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1761234019.533461:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533464:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234019.533465:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234019.533466:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 02000000:00000010:2.0:1761234019.533469:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000dd1dcb0e. 00000020:00000001:3.0:1761234019.533469:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234019.533475:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 02000000:00000001:2.0:1761234019.533477:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234019.533488:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000010:2.0:1761234019.533491:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000a1db4867. 00000020:00000001:3.0:1761234019.533492:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000010:2.0:1761234019.533500:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000053c991f. 00000100:00000040:3.0:1761234019.533500:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533522:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234019.533524:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533525:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234019.533527:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:3.0:1761234019.533529:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000200:2.0:1761234019.533530:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245225792, portal 25 00000100:00000001:3.0:1761234019.533531:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.533535:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1761234019.533536:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234019.533540:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234019.533541:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091110144 : -104717618441472 : ffffa0c2878d1300) 02000000:00000001:3.0:1761234019.533543:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533547:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234019.533548:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000040:2.0:1761234019.533552:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234044 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000008:00000001:3.0:1761234019.533554:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533560:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234019.533571:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f61de4c8. 00000100:00000001:2.0:1761234019.533572:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000040:3.0:1761234019.533576:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:3.0:1761234019.533584:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000040:2.0:1761234019.533585:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 02000000:00000001:3.0:1761234019.533586:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234019.533588:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000001d64cf84. 00000100:00000001:3.0:1761234019.533589:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1761234019.533595:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245225792, offset 0 02000000:00000001:3.0:1761234019.533595:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234019.533600:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234019.533604:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000df51b12a. 00000400:00000200:2.0:1761234019.533611:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:3.0:1761234019.533613:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234019.533619:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234019.533625:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234019.533626:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234019.533627:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000040:3.0:1761234019.533631:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f61de4c8] to pc [ptlrpcd_rcv+-1] req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234019.533636:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000001:3.0:1761234019.533664:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234019.533666:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000200:2.0:1761234019.533667:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1761234019.533672:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6498) 00000400:00000200:2.0:1761234019.533673:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234019.533681:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11745:11745:256:4294967295] 192.168.201.124@tcp LPNI seq info [11745:11745:8:4294967295] 00000400:00000200:2.0:1761234019.533692:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234019.533697:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234019.533702:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009b62b3c1 (tot 1676530). 00000800:00000200:2.0:1761234019.533708:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234019.533716:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:2.0:1761234019.533737:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.533739:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234019.533741:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.533743:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.533745:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234019.533751:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234019.533753:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1761234019.533756:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234019.533758:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234019.533759:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234019.533764:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234019.533769:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234019.533770:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234019.533772:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234019.533773:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234019.533774:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245225856:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234019.533777:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234019.533778:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234019.533779:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234019.533780:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234019.533783:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234019.533784:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234019.533786:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 0000000044d6aecc. 00000400:00000010:2.0:1761234019.533791:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000e8954a37. 00000100:00000200:2.0:1761234019.533794:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245225856, portal 10 00000100:00000001:2.0:1761234019.533796:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234019.533798:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091128384 : -104717618423232 : ffffa0c2878d5a40) 00000100:00000040:2.0:1761234019.533803:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234044 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234019.533807:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234019.533809:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234019.533810:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234019.533811:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245225856, offset 0 00000400:00000200:2.0:1761234019.533814:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234019.533818:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234019.533822:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234019.533823:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234019.533824:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234019.533826:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11746:11746:255:4294967295] 192.168.201.124@tcp LPNI seq info [11746:11746:7:4294967295] 00000400:00000200:2.0:1761234019.533831:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234019.533834:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234019.533837:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676778). 00000400:00000200:3.0:1761234019.533837:0:88984:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000800:00000200:2.0:1761234019.533883:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234019.533890:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:2.0:1761234019.533893:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.533895:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234019.533896:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.533909:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234019.533911:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234019.533916:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234019.533917:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234019.533919:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234019.533922:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234019.533924:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245225920:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234019.533926:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234019.533928:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234019.533928:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234019.533929:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234019.533931:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f7091f62. 02000000:00000001:2.0:1761234019.533933:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234019.533936:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000a8fa17c7. 00000400:00000010:2.0:1761234019.533939:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000ca0eefa8. 00000100:00000200:2.0:1761234019.533943:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245225920, portal 4 00000100:00000001:2.0:1761234019.533945:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234019.533947:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091124736 : -104717618426880 : ffffa0c2878d4c00) 00000100:00000040:2.0:1761234019.533951:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234044 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234019.533956:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234019.533959:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234019.533961:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234019.533963:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245225920, offset 0 00000400:00000200:2.0:1761234019.533969:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234019.533976:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234019.533981:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234019.533984:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234019.533987:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234019.533990:0:88984:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:2.0:1761234019.533991:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11747:11747:254:4294967295] 192.168.201.124@tcp LPNI seq info [11747:11747:6:4294967295] 00000400:00000200:2.0:1761234019.534015:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234019.534018:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234019.534023:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1677026). 00000800:00000200:2.0:1761234019.534029:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234019.534036:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:2.0:1761234019.534038:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.534040:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1761234019.534042:0:88984:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000100:00000001:2.0:1761234019.534043:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.534052:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.534056:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234019.534060:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234019.534062:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1761234019.534067:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234019.534069:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234019.534075:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234019.534078:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234020.044969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234020.044975:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234020.044976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234020.044981:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234020.044986:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234020.044989:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234020.044993:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234020.044995:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234020.044997:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234020.429004:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234020.429010:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234020.429019:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234020.429020:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234020.429025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234020.429027:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234020.429121:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234020.429125:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234020.429129:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234021.069010:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234021.069014:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234021.069015:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234021.069019:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234021.069023:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234021.069025:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234021.069027:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234021.069029:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234021.069031:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234021.452931:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234021.452937:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234021.452943:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234021.452944:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234021.452948:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234021.452949:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234021.453016:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234021.453019:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234021.453023:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234022.093025:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234022.093032:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234022.093034:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234022.093040:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234022.093047:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234022.093052:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234022.093055:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234022.093059:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234022.093063:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234022.477017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234022.477024:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234022.477031:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234022.477033:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234022.477038:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234022.477040:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234022.477151:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234022.477155:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234022.477158:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1761234022.605338:0:88984:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:2.0:1761234022.605420:0:88984:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1761234022.605435:0:88984:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:2.0:1761234022.605454:0:88984:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:2.0:1761234022.605460:0:88984:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009b62b3c1 (tot 1676778). 00000400:00000200:2.0:1761234022.605470:0:88984:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1761234022.605478:0:88984:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1761234022.605488:0:88984:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000001d64cf84 00000400:00000010:2.0:1761234022.605491:0:88984:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000001d64cf84. 00000100:00000001:2.0:1761234022.605500:0:88984:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234022.605512:0:88984:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234044 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234022.605571:0:88984:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234022.605576:0:88984:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234044 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234022.605605:0:88984:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234022.605608:0:88984:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:2.0:1761234022.605623:0:88984:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:2.0:1761234022.605626:0:88984:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676530). 00000400:00000200:2.0:1761234022.605630:0:88984:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1761234022.605636:0:88984:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000100:00000001:3.0:1761234022.605639:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1761234022.605641:0:88984:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234022.605645:0:88984:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:3.0:1761234022.605647:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234022.605648:0:88984:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234022.605655:0:88984:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234044 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234022.605656:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1761234022.605668:0:88984:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00100000:3.0:1761234022.605668:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234019/real 1761234022] req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234022.605675:0:88984:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234044 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234022.605707:0:88984:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234022.605710:0:88984:0:(events.c:93:request_out_callback()) Process leaving 00000400:00000001:3.0:1761234022.605712:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234022.605716:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000800:00000100:2.0:1761234022.605720:0:88984:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000400:00000010:3.0:1761234022.605721:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000a1db4867. 00000800:00000010:2.0:1761234022.605725:0:88984:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:3.0:1761234022.605730:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000053c991f 00000400:00000200:2.0:1761234022.605731:0:88984:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000010:3.0:1761234022.605735:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000053c991f. 00000400:00000200:2.0:1761234022.605740:0:88984:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000100:00000001:3.0:1761234022.605743:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000400:00000200:2.0:1761234022.605746:0:88984:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000100:00000200:3.0:1761234022.605751:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000010:2.0:1761234022.605752:0:88984:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234022.605758:0:88984:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234022.605766:0:88984:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234044 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234022.605770:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234022.605796:0:88984:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000001:3.0:1761234022.605798:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234022.605802:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234022.605805:0:88984:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234044 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234022.605807:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234022.605811:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234022.605820:0:88984:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234022.605821:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234022.605824:0:88984:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234022.605834:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234022.605841:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234022.605913:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234022.605919:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.605922:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234022.605924:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.605927:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234022.605931:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234022.605935:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234022.605941:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234022.605945:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234022.605950:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234022.605973:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245225792:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234022.605980:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234022.605982:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234022.605985:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000108ab0a5 x1846784245225792/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234022.605994:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234022.605999:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234022.606005:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000dd1dcb0e. 02000000:00000001:3.0:1761234022.606011:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234022.606015:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234022.606018:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234022.606022:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234022.606025:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000001c1b0e94. 02000000:00000001:3.0:1761234022.606031:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234022.606033:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234022.606035:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000108ab0a5. 00000100:00000001:3.0:1761234022.606041:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234022.606043:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606046:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234022.606049:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234022.606051:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234022.606053:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234022.606057:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234019/real 1761234022] req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234022.606068:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234022.606069:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234022.606072:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 0000000044d6aecc. 00000400:00000200:3.0:1761234022.606075:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000e8954a37 00000400:00000010:3.0:1761234022.606076:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000e8954a37. 00000100:00000001:3.0:1761234022.606080:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234022.606083:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234022.606092:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234022.606100:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234022.606101:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606104:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234022.606105:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234022.606109:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234022.606117:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234022.606120:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234022.606129:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234022.606132:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606135:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234022.606136:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606138:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234022.606140:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234022.606143:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234022.606147:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234022.606149:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234022.606153:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234022.606164:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245225856:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234022.606170:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234022.606173:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234019/real 1761234022] req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234022.606184:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234022.606185:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234022.606188:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000a8fa17c7. 00000400:00000200:3.0:1761234022.606190:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000ca0eefa8 00000400:00000010:3.0:1761234022.606193:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000ca0eefa8. 00000100:00000001:3.0:1761234022.606211:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234022.606213:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234022.606222:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234022.606229:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234022.606230:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606233:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234022.606234:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234022.606238:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234022.606245:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234022.606248:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234022.606256:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234022.606259:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606260:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234022.606262:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606263:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234022.606265:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234022.606267:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234022.606271:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234022.606274:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234022.606278:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234022.606285:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245225920:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234022.606290:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606292:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234022.606294:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002431276f x1846784245225856/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234022.606303:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234022.606304:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234022.606306:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062f3eded. 02000000:00000001:3.0:1761234022.606311:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234022.606312:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234022.606313:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234022.606316:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234022.606317:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234022.606323:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234022.606324:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234022.606325:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002431276f. 00000100:00000001:3.0:1761234022.606328:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234022.606330:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606332:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234022.606335:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f61de4c8 x1846784245225920/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234044 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234022.606342:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234022.606343:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234022.606345:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f7091f62. 02000000:00000001:3.0:1761234022.606348:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234022.606349:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234022.606350:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234022.606440:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234022.606442:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000df51b12a. 02000000:00000001:3.0:1761234022.606446:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234022.606448:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234022.606449:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f61de4c8. 00000100:00000001:3.0:1761234022.606454:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234022.606456:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606458:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234022.606469:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234022.606472:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234022.606475:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234022.606476:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234023.117021:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234023.117029:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234023.117031:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234023.117038:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234023.117046:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234023.117052:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234023.117056:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234023.117060:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234023.117065:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234023.500990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234023.500995:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234023.501002:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234023.501004:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234023.501008:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234023.501009:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234023.501153:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234023.501162:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234023.501170:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234024.140982:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234024.140987:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234024.140988:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234024.140992:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234024.140997:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234024.141000:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234024.141003:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234024.141005:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234024.141008:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234024.524954:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234024.524958:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234024.524965:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234024.524966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234024.524969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234024.524970:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234024.525042:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234024.525044:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234024.525047:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234024.652978:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234024.652989:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234024.652991:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234024.652993:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234024.652996:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234024.653001:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234024.653007:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6493 00000100:00080000:0.0:1761234024.653010:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234024.653014:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653017:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234024.653019:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653021:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234024.653025:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234024.653027:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234024.653029:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234024.653032:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234024.653034:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234024.653037:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234024.653039:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653041:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234024.653043:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653044:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234024.653047:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234024.653050:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653053:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234024.653055:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234024.653058:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234024.653061:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653063:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234024.653065:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234024.653072:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761234024.653075:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234024.653079:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234024.653080:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653083:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234024.653085:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234024.653088:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234024.653095:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761234024.653100:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234024.653106:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234024.653116:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234024.653119:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234024.653122:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234024.653144:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653146:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234024.653148:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234024.653151:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234024.653152:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234024.653153:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234024.653155:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234024.653157:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234024.653160:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6493 00000100:00080000:0.0:1761234024.653162:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234024.653164:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653166:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234024.653168:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653171:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234024.653173:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234024.653175:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234024.653177:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234024.653179:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234024.653182:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234024.653183:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234024.653187:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234024.653187:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1761234024.653191:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1761234024.653192:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1761234024.653193:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234024.653196:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1761234024.653197:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234024.653200:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00000040:3.0:1761234024.653204:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761234024.653205:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653209:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234024.653212:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234024.653217:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234024.653219:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1761234024.653220:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.653222:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653223:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234024.653228:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1761234024.653229:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1761234024.653232:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234024.653232:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653237:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234024.653240:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086a5741d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245225984:192.168.201.124@tcp:250: 00000100:00000010:0.0:1761234024.653241:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761234024.653245:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:3.0:1761234024.653247:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1761234024.653249:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234024.653252:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234024.653252:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1761234024.653256:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234024.653256:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234024.653258:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234024.653261:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234024.653261:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234024.653267:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000df51b12a. 02000000:00000010:0.0:1761234024.653268:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 02000000:00000001:3.0:1761234024.653272:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653274:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234024.653280:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00100000:0.0:1761234024.653281:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234024.653289:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1761234024.653289:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000001:0.0:1761234024.653291:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234024.653295:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234024.653298:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245225984, portal 25 00000100:00000001:3.0:1761234024.653303:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234024.653304:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246304256 : -104717463247360 : ffffa0c290cd2600) 00000100:00000040:3.0:1761234024.653314:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234049 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234024.653315:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653319:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234024.653320:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234024.653327:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234024.653328:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761234024.653330:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234024.653331:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234024.653333:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234024.653336:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6493 00000100:00000001:3.0:1761234024.653338:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00080000:0.0:1761234024.653339:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234024.653345:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000040:3.0:1761234024.653346:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653348:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234024.653351:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000400:00000010:3.0:1761234024.653351:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000001:0.0:1761234024.653353:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234024.653357:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000100:00000200:3.0:1761234024.653357:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245225984, offset 0 00000020:00000040:0.0:1761234024.653359:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234024.653362:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234024.653363:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761234024.653365:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761234024.653367:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000200:3.0:1761234024.653370:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000040:0.0:1761234024.653374:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653377:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234024.653379:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653381:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234024.653383:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761234024.653386:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234024.653388:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234024.653391:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1761234024.653391:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 02000000:00000001:0.0:1761234024.653395:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234024.653397:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653399:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234024.653401:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1761234024.653403:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000008:00000001:0.0:1761234024.653405:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653409:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234024.653409:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000010:0.0:1761234024.653413:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000400:00000200:3.0:1761234024.653415:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000020:00000040:0.0:1761234024.653418:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234024.653423:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000400:00000200:3.0:1761234024.653423:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11748:11748:256:4294967295] 192.168.201.124@tcp LPNI seq info [11748:11748:8:4294967295] 02000000:00000001:0.0:1761234024.653425:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653428:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234024.653431:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234024.653432:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234024.653437:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000400:00000200:3.0:1761234024.653439:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:0.0:1761234024.653442:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234024.653450:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.0:1761234024.653452:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234024.653460:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000100:00000001:0.0:1761234024.653461:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234024.653464:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234024.653469:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234024.653477:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234024.653479:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761234024.653483:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6503) 00000800:00000200:3.0:1761234024.653696:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234024.653701:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761234024.653707:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234024.653710:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1761234024.653714:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234024.653716:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234024.653721:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234024.653727:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234024.653733:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234024.653735:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234024.653740:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234024.653743:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234049 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.653753:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234024.653756:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234049 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.653763:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.653765:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234024.653773:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.653775:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234024.653778:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.653781:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234024.653785:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234024/real 1761234024] req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234024.653795:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234024.653797:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234024.653799:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234024.653801:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234024.653803:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234024.653805:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234024.653807:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234024.653821:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.653827:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234024.653829:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.653831:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234024.653833:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234024.653837:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.653962:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234024.653981:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234024.653993:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234024.653997:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.653999:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234024.654001:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654004:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234024.654122:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234024.654127:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234024.654132:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234024.654135:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234024.654140:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234024.654149:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a5741d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245225984:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234024.654155:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654159:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234024.654162:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000086a5741d x1846784245225984/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234024.654169:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234024.654173:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234024.654179:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000df51b12a. 02000000:00000001:3.0:1761234024.654185:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234024.654187:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234024.654189:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234024.654192:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234024.654195:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000005851ba37. 02000000:00000001:3.0:1761234024.654200:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234024.654202:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234024.654204:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000086a5741d. 00000100:00000001:3.0:1761234024.654209:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234024.654210:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654213:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654225:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234024.654227:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.654230:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234024.654233:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234024.654235:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234024.654239:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654248:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234024.654249:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234024.654253:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234024.654256:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234024.654259:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2788073 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226048:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234024.654263:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234024.654268:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234024.654270:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234024.654272:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234024.654281:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000df51b12a. 02000000:00000001:3.0:1761234024.654283:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234024.654291:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234024.654297:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234024.654305:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226048, portal 10 00000100:00000001:3.0:1761234024.654309:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234024.654310:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246317632 : -104717463233984 : ffffa0c290cd5a40) 00000100:00000040:3.0:1761234024.654318:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234049 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654325:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234024.654328:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234024.654331:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234024.654334:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245226048, offset 0 00000400:00000200:3.0:1761234024.654341:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234024.654358:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234024.654366:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234024.654371:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234024.654374:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234024.654379:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11749:11749:256:4294967295] 192.168.201.124@tcp LPNI seq info [11749:11749:8:4294967295] 00000400:00000200:3.0:1761234024.654390:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234024.654397:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234024.654403:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234024.654410:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234024.654413:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761234024.654419:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234024.654423:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1761234024.654428:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234024.654430:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234024.654435:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234024.654441:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234024.654455:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234024.654461:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234024.654465:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234024.654469:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234049 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654478:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234024.654480:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234049 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654486:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.654488:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234024.654497:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.654500:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234024.654502:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.654505:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234024.654508:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234024/real 1761234024] req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234024.654518:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234024.654519:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234024.654521:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234024.654523:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234024.654525:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234024.654527:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234024.654530:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234024.654538:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654545:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234024.654562:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654564:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234024.654565:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234024.654569:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654576:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234024.654579:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234024.654586:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234024.654588:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654590:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234024.654591:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654593:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234024.654594:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234024.654597:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234024.654601:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234024.654603:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234024.654606:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234024.654613:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2788073 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226048:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234024.654618:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234024.654621:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654628:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234024.654629:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234024.654632:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234024.654633:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234024.654636:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a0dd9b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226112:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234024.654639:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234024.654641:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234024.654643:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234024.654645:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234024.654648:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234024.654650:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234024.654652:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234024.654655:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234024.654657:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226112, portal 4 00000100:00000001:3.0:1761234024.654659:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234024.654660:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246322496 : -104717463229120 : ffffa0c290cd6d40) 00000100:00000040:3.0:1761234024.654664:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234049 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654668:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234024.654670:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234024.654671:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234024.654672:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245226112, offset 0 00000400:00000200:3.0:1761234024.654676:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234024.654680:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234024.654683:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234024.654684:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234024.654686:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234024.654688:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11750:11750:256:4294967295] 192.168.201.124@tcp LPNI seq info [11750:11750:8:4294967295] 00000400:00000200:3.0:1761234024.654694:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234024.654698:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234024.654701:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234024.654705:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234024.654707:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761234024.654711:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234024.654713:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1761234024.654717:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234024.654719:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234024.654722:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234024.654725:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234024.654728:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234024.654729:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234024.654731:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234024.654734:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234049 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654741:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234024.654743:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234049 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654749:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.654750:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234024.654754:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.654755:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234024.654758:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.654760:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234024.654762:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234024/real 1761234024] req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234024.654770:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234024.654771:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234024.654773:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234024.654775:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234024.654776:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234024.654778:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234024.654780:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234024.654786:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654792:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234024.654793:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654795:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234024.654796:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234024.654798:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234024.654805:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234024.654808:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234024.654814:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234024.654816:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654818:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234024.654819:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654821:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234024.654822:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234024.654824:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234024.654827:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234024.654828:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234024.654831:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234024.654837:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a0dd9b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226112:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234024.654875:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654877:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234024.654880:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000c2788073 x1846784245226048/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234024.654886:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234024.654888:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234024.654890:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000df51b12a. 02000000:00000001:3.0:1761234024.654894:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234024.654895:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234024.654896:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234024.654898:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234024.654899:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000007c64e188. 02000000:00000001:3.0:1761234024.654902:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234024.654903:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234024.654904:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000c2788073. 00000100:00000001:3.0:1761234024.654907:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234024.654908:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654910:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234024.654912:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007a0dd9b5 x1846784245226112/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234049 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234024.654918:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234024.654919:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234024.654919:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234024.654921:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234024.654922:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234024.654923:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234024.654925:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234024.654926:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:3.0:1761234024.654928:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234024.654929:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234024.654930:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007a0dd9b5. 00000100:00000001:3.0:1761234024.654932:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234024.654933:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654934:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234024.654940:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234024.654942:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234024.654944:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234024.654945:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234025.164962:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234025.164966:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234025.164967:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234025.164971:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234025.164975:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234025.164978:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234025.164979:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234025.164981:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234025.164983:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234025.548902:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234025.548906:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234025.548910:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234025.548911:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234025.548914:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234025.548915:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234025.548981:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234025.548984:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234025.548987:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234026.189019:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234026.189023:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234026.189025:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234026.189029:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234026.189032:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234026.189036:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234026.189037:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234026.189039:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234026.189041:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234026.572973:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234026.572978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234026.572984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234026.572986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234026.572990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234026.572991:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234026.573094:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234026.573098:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234026.573101:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234027.212963:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234027.212968:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234027.212969:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234027.212975:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234027.212979:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234027.212983:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234027.212985:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234027.212987:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234027.212990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234027.597020:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234027.597029:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234027.597040:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234027.597041:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234027.597047:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234027.597049:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234027.597137:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234027.597142:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234027.597148:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234028.236984:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234028.236989:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234028.236991:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234028.236995:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234028.237001:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234028.237005:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234028.237007:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234028.237009:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234028.237012:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234028.621025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234028.621031:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234028.621038:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234028.621039:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234028.621042:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234028.621043:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234028.621157:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234028.621161:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234028.621166:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234029.260956:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234029.260964:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234029.260967:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234029.260974:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234029.260981:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234029.260986:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234029.260989:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234029.260992:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234029.260996:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.644994:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234029.645001:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234029.645003:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234029.645006:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645011:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234029.645013:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645014:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645015:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645020:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234029.645022:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234029.645023:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234029.645027:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234029.645028:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234029.645029:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645031:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234029.645032:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645033:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645034:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645038:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234029.645039:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234029.645041:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234029.645043:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234029.645044:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234029.645045:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645046:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234029.645048:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645049:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234029.645051:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234029.645118:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234029.645124:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234029.645128:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761234029.773014:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234029.773024:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234029.773026:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:2.0:1761234029.773028:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234029.773030:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234029.773034:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234029.773040:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6498 00000100:00080000:2.0:1761234029.773042:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234029.773045:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773047:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234029.773048:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773050:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234029.773053:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234029.773054:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:2.0:1761234029.773056:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234029.773059:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:2.0:1761234029.773060:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:2.0:1761234029.773062:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:2.0:1761234029.773064:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773065:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234029.773066:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773068:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234029.773069:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:2.0:1761234029.773071:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773073:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234029.773075:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234029.773077:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234029.773079:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234029.773082:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234029.773083:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234029.773091:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d4d64bb. 00000020:00000040:2.0:1761234029.773094:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:2.0:1761234029.773097:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234029.773098:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234029.773101:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234029.773103:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234029.773106:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234029.773114:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000116ca75c. 00000100:00000001:2.0:1761234029.773118:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234029.773124:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234029.773131:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234029.773133:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234029.773135:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d4d64bb] to pc [ptlrpcd_rcv+-1] req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234029.773155:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234029.773156:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234029.773158:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234029.773160:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234029.773161:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1761234029.773162:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234029.773163:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234029.773165:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234029.773166:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6498 00000100:00080000:2.0:1761234029.773167:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234029.773169:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773170:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234029.773171:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773172:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234029.773173:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234029.773175:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:2.0:1761234029.773177:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234029.773179:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:2.0:1761234029.773180:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:2.0:1761234029.773181:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:2.0:1761234029.773183:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773186:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234029.773190:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773192:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234029.773194:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:2.0:1761234029.773196:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773198:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234029.773199:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234029.773203:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234029.773204:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773206:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234029.773207:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234029.773211:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234029.773214:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234029.773216:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773216:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000010:2.0:1761234029.773219:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d82b1f3. 00000020:00000040:2.0:1761234029.773221:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:3.0:1761234029.773222:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1761234029.773223:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234029.773225:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234029.773226:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234029.773227:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234029.773255:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234029.773261:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078016eab. 00000100:00000040:3.0:1761234029.773265:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234029.773267:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234029.773277:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234029.773345:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234029.773346:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234029.773349:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773350:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234029.773371:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1761234029.773375:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d82b1f3] to pc [ptlrpcd_rcv+-1] req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234029.773375:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234029.773388:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d4d64bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226176:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234029.773397:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773400:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1761234029.773401:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234029.773403:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234029.773416:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234029.773420:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1761234029.773421:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234029.773422:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1761234029.773423:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1761234029.773426:0:468:0:(import.c:519:import_select_connection()) Process entered 02000000:00000001:3.0:1761234029.773426:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1761234029.773431:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6498 02000000:00000001:3.0:1761234029.773432:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1761234029.773436:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234029.773439:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773440:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234029.773441:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773442:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234029.773444:0:468:0:(genops.c:899:class_conn2export()) Process entered 02000000:00000010:3.0:1761234029.773444:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 00000020:00000040:2.0:1761234029.773446:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 02000000:00000001:3.0:1761234029.773448:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234029.773467:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000400:00000010:3.0:1761234029.773469:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000020:00000040:2.0:1761234029.773470:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:2.0:1761234029.773474:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:2.0:1761234029.773476:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234029.773479:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000010:3.0:1761234029.773479:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000001:2.0:1761234029.773485:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234029.773486:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773487:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234029.773489:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00000200:3.0:1761234029.773490:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226176, portal 25 00000100:00080000:2.0:1761234029.773491:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773494:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773495:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1761234029.773498:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1761234029.773498:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090818880 : -104717618732736 : ffffa0c28788a140) 02000000:00000001:2.0:1761234029.773503:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1761234029.773510:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234054 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234029.773511:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234029.773522:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234029.773523:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234029.773532:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773534:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:2.0:1761234029.773535:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234029.773543:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000010:2.0:1761234029.773544:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000069aeaefe. 00000400:00000010:3.0:1761234029.773547:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000020:00000040:2.0:1761234029.773548:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000200:3.0:1761234029.773551:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245226176, offset 0 02000000:00000001:2.0:1761234029.773552:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234029.773556:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234029.773560:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234029.773562:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1761234029.773562:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:2.0:1761234029.773568:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234029.773574:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006ca8bb96. 00000100:00000001:2.0:1761234029.773580:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234029.773582:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00100000:2.0:1761234029.773590:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234029.773593:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234029.773601:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234029.773608:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000001:2.0:1761234029.773615:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1761234029.773616:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11751:11751:256:4294967295] 192.168.201.124@tcp LPNI seq info [11751:11751:8:4294967295] 00000100:00000001:2.0:1761234029.773617:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234029.773628:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000069aeaefe] to pc [ptlrpcd_rcv+-1] req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234029.773644:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:2.0:1761234029.773664:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234029.773670:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:3.0:1761234029.773670:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234029.773678:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000100:00000040:2.0:1761234029.773679:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6508) 00000800:00000200:3.0:1761234029.773694:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234029.773709:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761234029.773716:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234029.773719:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1761234029.773725:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234029.773728:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234029.773735:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234029.773740:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234029.773745:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234029.773747:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234029.773752:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234029.773755:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234054 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.773763:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234029.773765:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234054 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.773770:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773771:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234029.773777:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773778:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234029.773780:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773782:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234029.773785:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234029/real 1761234029] req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234029.773792:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234029.773793:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234029.773795:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234029.773796:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234029.773798:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234029.773799:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234029.773801:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234029.773805:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.773809:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234029.773811:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.773812:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234029.773814:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234029.773816:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.773820:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234029.773823:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234029.773827:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234029.773829:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.773830:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234029.773830:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.773833:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234029.773834:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234029.773836:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234029.773839:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234029.773841:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234029.773873:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234029.773879:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d4d64bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226176:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234029.773884:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.773885:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234029.773886:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007d4d64bb x1846784245226176/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234029.773891:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234029.773893:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234029.773896:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234029.773898:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234029.773900:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234029.773902:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234029.773904:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234029.773906:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000116ca75c. 02000000:00000001:3.0:1761234029.773911:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234029.773912:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234029.773913:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007d4d64bb. 00000100:00000001:3.0:1761234029.773917:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234029.773918:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.773920:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.773932:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234029.773933:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.773936:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234029.773938:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234029.773939:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234029.773943:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.773950:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234029.773951:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234029.773953:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234029.773956:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234029.773959:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d82b1f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226240:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234029.773962:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234029.773964:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234029.773965:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234029.773967:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234029.773970:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234029.773971:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234029.773975:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234029.773979:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234029.773982:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226240, portal 10 00000100:00000001:3.0:1761234029.773985:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234029.773986:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090827392 : -104717618724224 : ffffa0c28788c280) 00000100:00000040:3.0:1761234029.773993:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234054 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.773999:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234029.774002:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234029.774004:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234029.774006:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245226240, offset 0 00000400:00000200:3.0:1761234029.774011:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234029.774016:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234029.774020:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234029.774022:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234029.774025:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234029.774026:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11752:11752:256:4294967295] 192.168.201.124@tcp LPNI seq info [11752:11752:8:4294967295] 00000400:00000200:3.0:1761234029.774032:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234029.774035:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234029.774038:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234029.774040:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234029.774042:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761234029.774045:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234029.774047:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1761234029.774049:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234029.774050:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234029.774053:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234029.774055:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234029.774057:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234029.774058:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234029.774059:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234029.774061:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234054 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774067:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234029.774068:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234054 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774076:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.774077:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234029.774080:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.774081:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234029.774082:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.774083:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234029.774085:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234029/real 1761234029] req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234029.774090:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234029.774091:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234029.774092:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234029.774093:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234029.774093:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234029.774095:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234029.774096:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234029.774101:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774104:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234029.774105:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774106:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234029.774106:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234029.774108:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774111:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234029.774113:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234029.774117:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234029.774118:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774119:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234029.774120:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774122:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234029.774124:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234029.774125:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234029.774127:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234029.774129:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234029.774131:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234029.774134:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d82b1f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226240:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234029.774136:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234029.774138:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774142:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234029.774143:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234029.774144:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234029.774145:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234029.774147:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069aeaefe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226304:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234029.774149:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234029.774151:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234029.774152:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234029.774153:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234029.774155:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234029.774156:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234029.774158:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234029.774160:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234029.774162:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226304, portal 4 00000100:00000001:3.0:1761234029.774164:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234029.774164:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090833472 : -104717618718144 : ffffa0c28788da40) 00000100:00000040:3.0:1761234029.774168:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234054 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774172:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234029.774174:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234029.774175:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234029.774177:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245226304, offset 0 00000400:00000200:3.0:1761234029.774179:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234029.774182:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234029.774184:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234029.774186:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234029.774187:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234029.774189:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11753:11753:256:4294967295] 192.168.201.124@tcp LPNI seq info [11753:11753:8:4294967295] 00000400:00000200:3.0:1761234029.774195:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234029.774197:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234029.774198:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234029.774201:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234029.774202:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1761234029.774205:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234029.774206:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1761234029.774208:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234029.774210:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234029.774211:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234029.774213:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234029.774215:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234029.774216:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234029.774217:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234029.774218:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234054 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774222:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234029.774223:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234054 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774227:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.774228:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234029.774229:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.774230:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234029.774231:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.774233:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234029.774234:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234029/real 1761234029] req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234029.774239:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234029.774240:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234029.774240:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234029.774241:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234029.774242:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234029.774243:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234029.774244:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234029.774247:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774251:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234029.774251:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774252:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234029.774253:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234029.774254:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234029.774257:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234029.774259:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234029.774262:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234029.774263:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774264:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234029.774265:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774266:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234029.774267:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234029.774268:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234029.774270:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234029.774271:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234029.774273:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234029.774278:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069aeaefe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226304:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234029.774280:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774281:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234029.774282:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007d82b1f3 x1846784245226240/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234029.774286:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234029.774287:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234029.774287:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234029.774289:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234029.774289:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234029.774290:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234029.774291:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234029.774292:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078016eab. 02000000:00000001:3.0:1761234029.774294:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234029.774294:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234029.774295:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007d82b1f3. 00000100:00000001:3.0:1761234029.774299:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234029.774300:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774301:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234029.774303:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000069aeaefe x1846784245226304/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234054 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234029.774307:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234029.774307:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234029.774308:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234029.774310:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234029.774311:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234029.774312:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234029.774313:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234029.774314:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006ca8bb96. 02000000:00000001:3.0:1761234029.774316:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234029.774316:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234029.774318:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000069aeaefe. 00000100:00000001:3.0:1761234029.774321:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234029.774322:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774323:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234029.774326:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234029.774326:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234029.774328:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234029.774329:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234030.284945:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234030.284953:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234030.284956:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234030.284961:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234030.284968:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234030.284971:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234030.284973:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234030.284976:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234030.284980:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234030.668978:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234030.668986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234030.668995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234030.668997:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234030.669003:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234030.669005:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234030.669101:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234030.669104:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234030.669107:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234031.308940:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234031.308945:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234031.308947:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234031.308952:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234031.308957:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234031.308960:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234031.308962:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234031.308964:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234031.308966:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234031.693009:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234031.693014:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234031.693020:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234031.693022:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234031.693025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234031.693028:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234031.693127:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234031.693130:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234031.693133:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234032.332973:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234032.332982:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234032.332984:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234032.332990:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234032.332997:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234032.333001:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234032.333003:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234032.333006:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234032.333008:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234032.716956:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234032.716960:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234032.716964:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234032.716965:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234032.716968:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234032.716969:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234032.717030:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234032.717033:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234032.717035:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234033.356912:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234033.356917:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234033.356918:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234033.356921:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234033.356925:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234033.356928:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234033.356930:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234033.356931:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234033.356933:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234033.740970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234033.740974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234033.740979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234033.740980:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234033.740982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234033.740983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234033.741044:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234033.741047:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234033.741050:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234034.380950:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234034.380954:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234034.380956:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234034.380959:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234034.380964:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234034.380967:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234034.380970:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234034.380971:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234034.380974:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234034.764952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234034.764957:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234034.764963:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234034.764965:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234034.764968:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234034.764969:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234034.765081:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234034.765085:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234034.765089:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234034.892952:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234034.892962:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234034.892964:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234034.892968:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234034.892971:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234034.892976:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234034.892983:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6503 00000100:00080000:1.0:1761234034.892986:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234034.892988:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.892990:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234034.892991:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.892993:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234034.892996:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234034.892997:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234034.892999:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234034.893002:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234034.893003:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234034.893005:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234034.893007:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893008:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234034.893009:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893010:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234034.893012:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234034.893014:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893015:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234034.893018:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234034.893020:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234034.893021:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893023:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234034.893024:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234034.893032:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761234034.893034:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234034.893037:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234034.893038:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893040:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234034.893042:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234034.893045:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234034.893051:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761234034.893057:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234034.893062:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234034.893069:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234034.893071:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234034.893073:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234034.893090:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893092:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234034.893093:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234034.893095:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234034.893096:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234034.893097:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234034.893098:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234034.893099:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234034.893101:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6503 00000100:00080000:1.0:1761234034.893102:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234034.893103:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893104:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234034.893105:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893106:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234034.893107:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234034.893108:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234034.893108:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234034.893109:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234034.893110:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234034.893111:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234034.893113:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893114:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234034.893115:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893117:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234034.893119:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234034.893121:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893122:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234034.893123:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234034.893127:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234034.893128:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893130:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234034.893134:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234034.893136:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893138:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234034.893141:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761234034.893142:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234034.893144:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234034.893144:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893145:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234034.893146:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234034.893147:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234034.893149:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761234034.893151:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234034.893154:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893154:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234034.893162:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234034.893163:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761234034.893164:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234034.893167:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893169:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1761234034.893175:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893176:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234034.893178:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234034.893180:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234034.893181:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000040:3.0:1761234034.893183:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234034.893184:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234034.893185:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234034.893190:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234034.893194:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6503 00000100:00080000:1.0:1761234034.893197:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234034.893203:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893207:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234034.893211:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893215:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:3.0:1761234034.893218:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1761234034.893219:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234034.893222:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000100:00000001:3.0:1761234034.893223:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234034.893225:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234034.893227:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234034.893229:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:3.0:1761234034.893230:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1761234034.893231:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234034.893235:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000001:3.0:1761234034.893237:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893238:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234034.893241:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893243:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00100000:3.0:1761234034.893247:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226368:192.168.201.124@tcp:250: 00000020:00000040:1.0:1761234034.893248:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234034.893257:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234034.893263:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234034.893265:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1761234034.893266:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761234034.893267:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234034.893270:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893271:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234034.893272:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1761234034.893273:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:1.0:1761234034.893274:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893276:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234034.893278:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234034.893279:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761234034.893282:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:3.0:1761234034.893284:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1761234034.893285:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234034.893286:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893287:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234034.893288:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234034.893289:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234034.893293:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 02000000:00000010:3.0:1761234034.893295:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 00000100:00000001:1.0:1761234034.893296:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234034.893303:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234034.893303:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234034.893314:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234034.893315:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234034.893318:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00000040:1.0:1761234034.893319:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234034.893330:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234034.893331:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000001:1.0:1761234034.893333:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234034.893337:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6514) 00000100:00000200:3.0:1761234034.893345:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226368, portal 25 00000100:00000001:3.0:1761234034.893351:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234034.893365:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039807616 : -104717669744000 : ffffa0c2847e4280) 00000100:00000040:3.0:1761234034.893374:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234059 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893382:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234034.893387:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234034.893390:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234034.893393:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245226368, offset 0 00000400:00000200:3.0:1761234034.893402:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234034.893422:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234034.893432:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234034.893436:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234034.893439:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234034.893443:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11754:11754:256:4294967295] 192.168.201.124@tcp LPNI seq info [11754:11754:8:4294967295] 00000400:00000200:3.0:1761234034.893455:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234034.893461:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234034.893467:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234034.893476:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234034.893480:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1761234034.893487:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234034.893490:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1761234034.893495:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234034.893498:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234034.893504:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234034.893510:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234034.893515:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234034.893517:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234034.893522:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234034.893525:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234059 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893535:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234034.893538:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234059 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893550:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.893552:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234034.893559:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.893561:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234034.893564:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.893567:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234034.893570:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234034/real 1761234034] req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234034.893580:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234034.893583:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234034.893585:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234034.893588:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234034.893590:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234034.893592:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234034.893596:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234034.893604:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893611:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234034.893613:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.893615:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234034.893617:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234034.893620:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893627:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234034.893634:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234034.893640:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234034.893643:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.893644:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234034.893645:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.893648:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234034.893650:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234034.893653:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234034.893658:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234034.893661:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234034.893664:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234034.893671:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226368:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234034.893676:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.893677:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234034.893680:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a014c997 x1846784245226368/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234034.893686:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234034.893688:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234034.893691:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234034.893695:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234034.893697:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234034.893699:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234034.893701:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234034.893703:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:3.0:1761234034.893707:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234034.893709:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234034.893711:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a014c997. 00000100:00000001:3.0:1761234034.893714:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234034.893715:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.893718:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.893728:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234034.893730:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.893732:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234034.893733:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234034.893735:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234034.893738:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893744:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234034.893746:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234034.893749:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234034.893751:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234034.893753:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226432:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234034.893756:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234034.893758:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234034.893759:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234034.893761:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234034.893763:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234034.893765:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234034.893768:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234034.893771:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234034.893774:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226432, portal 10 00000100:00000001:3.0:1761234034.893775:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234034.893777:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039814912 : -104717669736704 : ffffa0c2847e5f00) 00000100:00000040:3.0:1761234034.893782:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234059 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893788:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234034.893791:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234034.893793:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234034.893795:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245226432, offset 0 00000400:00000200:3.0:1761234034.893799:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234034.893803:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234034.893807:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234034.893809:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234034.893811:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234034.893814:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11755:11755:256:4294967295] 192.168.201.124@tcp LPNI seq info [11755:11755:8:4294967295] 00000400:00000200:3.0:1761234034.893822:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234034.893826:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234034.893829:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234034.893833:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234034.893836:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1761234034.893840:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234034.893842:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1761234034.893882:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234034.893884:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234034.893887:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234034.893890:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234034.893893:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234034.893894:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234034.893896:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234034.893899:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234059 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893907:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234034.893909:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234059 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893914:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.893916:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234034.893919:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.893920:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234034.893922:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.893924:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234034.893926:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234034/real 1761234034] req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234034.893934:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234034.893935:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234034.893936:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234034.893938:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234034.893940:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234034.893942:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234034.893944:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234034.893951:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893956:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234034.893957:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.893959:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234034.893961:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234034.893964:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.893969:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234034.893972:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234034.893978:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234034.893981:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.893983:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234034.893984:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.893985:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234034.893987:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234034.893989:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234034.893992:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234034.893994:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234034.893996:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234034.894003:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226432:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234034.894007:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234034.894009:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.894015:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234034.894017:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234034.894019:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234034.894020:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234034.894022:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226496:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234034.894026:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234034.894027:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234034.894029:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234034.894031:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234034.894034:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234034.894035:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234034.894038:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234034.894042:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234034.894044:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226496, portal 4 00000100:00000001:3.0:1761234034.894046:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234034.894047:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039803968 : -104717669747648 : ffffa0c2847e3440) 00000100:00000040:3.0:1761234034.894051:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234059 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.894057:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234034.894059:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234034.894061:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234034.894062:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245226496, offset 0 00000400:00000200:3.0:1761234034.894066:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234034.894071:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234034.894074:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234034.894077:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234034.894078:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234034.894081:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11756:11756:256:4294967295] 192.168.201.124@tcp LPNI seq info [11756:11756:8:4294967295] 00000400:00000200:3.0:1761234034.894088:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234034.894092:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234034.894095:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234034.894098:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234034.894100:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1761234034.894104:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234034.894106:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1761234034.894109:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234034.894111:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234034.894113:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234034.894116:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234034.894119:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234034.894121:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234034.894123:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234034.894125:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234059 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.894131:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234034.894133:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234059 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.894139:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.894141:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234034.894143:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.894145:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234034.894146:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.894148:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234034.894150:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234034/real 1761234034] req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234034.894157:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234034.894158:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234034.894159:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234034.894161:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234034.894162:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234034.894164:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234034.894167:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234034.894173:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.894178:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234034.894179:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.894181:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234034.894182:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234034.894184:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234034.894190:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234034.894193:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234034.894199:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234034.894202:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.894203:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234034.894204:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.894205:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234034.894206:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234034.894208:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234034.894211:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234034.894213:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234034.894215:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234034.894222:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226496:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234034.894225:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.894227:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234034.894228:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000632be758 x1846784245226432/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234034.894234:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234034.894236:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234034.894237:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234034.894239:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234034.894240:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234034.894241:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234034.894243:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234034.894245:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:3.0:1761234034.894247:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234034.894248:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234034.894249:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000632be758. 00000100:00000001:3.0:1761234034.894252:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234034.894253:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.894255:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234034.894257:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000bb0f086 x1846784245226496/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234059 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234034.894263:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234034.894264:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234034.894265:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234034.894267:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234034.894268:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234034.894269:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234034.894271:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234034.894271:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006d8d01a9. 02000000:00000001:3.0:1761234034.894273:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234034.894275:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234034.894276:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000bb0f086. 00000100:00000001:3.0:1761234034.894277:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234034.894278:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.894279:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234034.894282:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234034.894283:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234034.894284:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234034.894286:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234035.405006:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234035.405012:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234035.405014:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234035.405020:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234035.405027:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234035.405031:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234035.405034:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234035.405037:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234035.405040:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234035.789027:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234035.789035:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234035.789043:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234035.789044:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234035.789050:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234035.789051:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234035.789187:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234035.789193:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234035.789197:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234036.428957:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234036.428964:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234036.428965:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234036.428971:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234036.428976:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234036.428979:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234036.428982:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234036.428984:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234036.428988:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234036.812928:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234036.812933:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234036.812939:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234036.812941:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234036.812944:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234036.812945:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234036.812987:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234036.812991:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234036.812993:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234037.452957:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234037.452963:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234037.452965:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234037.452970:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234037.452976:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234037.452980:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234037.452982:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234037.452985:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234037.452987:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234037.837015:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234037.837021:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234037.837028:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234037.837029:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234037.837035:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234037.837036:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234037.837122:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234037.837124:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234037.837127:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234038.476989:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234038.476996:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234038.476997:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234038.477002:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234038.477007:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234038.477011:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234038.477013:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234038.477015:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234038.477019:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234038.860942:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234038.860947:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234038.860954:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234038.860955:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234038.860958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234038.860959:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234038.861005:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234038.861008:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234038.861010:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234039.501025:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234039.501030:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234039.501032:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234039.501037:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234039.501042:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234039.501046:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234039.501048:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234039.501051:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234039.501053:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885029:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234039.885041:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234039.885044:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234039.885048:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885057:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234039.885060:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885064:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885067:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885074:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234039.885078:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234039.885080:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234039.885087:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234039.885089:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234039.885090:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885092:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234039.885093:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885094:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885095:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885099:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234039.885100:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234039.885101:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234039.885103:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234039.885104:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234039.885105:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885106:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234039.885107:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885108:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234039.885109:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234039.885187:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234039.885190:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234039.885193:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234040.012964:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234040.012977:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234040.012979:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234040.012982:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234040.012985:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234040.012991:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234040.012998:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6509 00000100:00080000:1.0:1761234040.013002:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234040.013005:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013008:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234040.013009:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013011:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234040.013014:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234040.013016:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234040.013018:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234040.013021:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234040.013023:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234040.013024:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234040.013026:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013027:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234040.013028:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013030:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234040.013032:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234040.013035:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013036:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234040.013039:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234040.013042:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234040.013044:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013046:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234040.013047:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234040.013057:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761234040.013060:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234040.013064:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234040.013065:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013068:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234040.013071:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234040.013075:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234040.013082:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761234040.013089:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234040.013096:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234040.013104:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234040.013106:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234040.013109:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234040.013128:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013129:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234040.013131:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234040.013133:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234040.013134:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234040.013135:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234040.013135:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234040.013137:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234040.013139:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6509 00000100:00080000:1.0:1761234040.013140:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234040.013141:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013142:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234040.013143:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013144:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234040.013145:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234040.013146:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234040.013146:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234040.013148:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234040.013149:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234040.013150:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234040.013151:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013152:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234040.013153:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013154:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234040.013155:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234040.013157:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013159:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234040.013160:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234040.013165:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234040.013166:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013168:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234040.013171:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234040.013174:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013175:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234040.013181:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761234040.013182:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234040.013183:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234040.013184:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013185:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234040.013187:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234040.013188:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234040.013190:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234040.013193:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234040.013196:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234040.013201:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234040.013202:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234040.013204:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013207:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234040.013208:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013211:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234040.013212:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234040.013213:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761234040.013214:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234040.013216:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234040.013216:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1761234040.013217:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234040.013219:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234040.013221:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234040.013225:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6509 00000100:00000040:3.0:1761234040.013225:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761234040.013229:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234040.013233:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234040.013237:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234040.013240:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234040.013242:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234040.013243:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:3.0:1761234040.013245:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234040.013248:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234040.013250:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 02000000:00000001:3.0:1761234040.013250:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1761234040.013252:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 02000000:00000001:3.0:1761234040.013254:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1761234040.013269:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00100000:3.0:1761234040.013273:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002b682c08 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226560:192.168.201.124@tcp:250: 00000020:00000001:1.0:1761234040.013275:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234040.013280:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:3.0:1761234040.013282:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1761234040.013287:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000001:3.0:1761234040.013287:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234040.013290:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013291:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1761234040.013293:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1761234040.013296:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 02000000:00000010:3.0:1761234040.013299:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 00000100:00000001:1.0:1761234040.013300:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 02000000:00000001:3.0:1761234040.013303:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1761234040.013305:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234040.013310:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000010:3.0:1761234040.013311:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00000001:1.0:1761234040.013312:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234040.013315:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234040.013318:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1761234040.013320:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 02000000:00000001:1.0:1761234040.013321:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013325:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234040.013328:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000200:3.0:1761234040.013328:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226560, portal 25 00000008:00000001:1.0:1761234040.013332:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013333:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234040.013335:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039801536 : -104717669750080 : ffffa0c2847e2ac0) 00000100:00000001:1.0:1761234040.013336:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234040.013342:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234065 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000010:1.0:1761234040.013344:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761234040.013348:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000001:3.0:1761234040.013352:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 02000000:00000001:1.0:1761234040.013354:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234040.013357:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234040.013357:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234040.013361:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000001:1.0:1761234040.013362:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234040.013365:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:3.0:1761234040.013365:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245226560, offset 0 02000000:00000001:1.0:1761234040.013369:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234040.013372:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000010:1.0:1761234040.013375:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761234040.013380:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234040.013385:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234040.013391:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:1.0:1761234040.013400:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234040.013401:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234040.013403:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000040:1.0:1761234040.013408:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234040.013410:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234040.013415:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234040.013420:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11757:11757:256:4294967295] 192.168.201.124@tcp LPNI seq info [11757:11757:8:4294967295] 00000100:00000001:1.0:1761234040.013421:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234040.013425:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234040.013430:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6519) 00000400:00000200:3.0:1761234040.013435:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234040.013441:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234040.013446:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234040.013454:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234040.013457:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1761234040.013463:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234040.013466:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1761234040.013470:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234040.013472:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234040.013478:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234040.013482:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234040.013486:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234040.013488:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234040.013491:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234040.013494:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234065 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013501:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234040.013503:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234065 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013508:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013509:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234040.013516:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013517:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234040.013519:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013522:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234040.013524:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234040/real 1761234040] req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234040.013531:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234040.013532:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234040.013534:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234040.013535:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234040.013536:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234040.013537:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234040.013538:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234040.013543:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013547:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234040.013548:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013550:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234040.013552:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234040.013554:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013558:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234040.013561:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234040.013566:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234040.013568:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013569:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234040.013570:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013572:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234040.013573:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234040.013575:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234040.013577:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234040.013578:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234040.013580:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234040.013585:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002b682c08 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226560:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234040.013588:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234040.013589:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013593:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234040.013594:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234040.013596:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234040.013597:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234040.013598:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226624:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234040.013600:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234040.013601:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234040.013602:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234040.013603:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234040.013605:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234040.013607:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234040.013609:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234040.013611:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234040.013613:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226624, portal 10 00000100:00000001:3.0:1761234040.013614:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234040.013615:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039803968 : -104717669747648 : ffffa0c2847e3440) 00000100:00000040:3.0:1761234040.013619:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234065 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013623:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234040.013625:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234040.013626:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234040.013627:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245226624, offset 0 00000400:00000200:3.0:1761234040.013630:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234040.013632:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234040.013634:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234040.013635:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234040.013637:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234040.013638:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11758:11758:256:4294967295] 192.168.201.124@tcp LPNI seq info [11758:11758:8:4294967295] 00000400:00000200:3.0:1761234040.013644:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234040.013646:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234040.013648:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234040.013650:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234040.013652:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1761234040.013654:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234040.013656:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1761234040.013658:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234040.013659:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234040.013661:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234040.013663:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234040.013665:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234040.013666:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234040.013667:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234040.013668:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234065 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013673:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234040.013675:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234065 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013679:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013680:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234040.013682:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013682:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234040.013683:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013684:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234040.013686:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234040/real 1761234040] req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234040.013689:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234040.013690:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234040.013691:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234040.013691:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234040.013692:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234040.013693:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234040.013694:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234040.013698:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013702:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234040.013703:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013704:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234040.013705:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234040.013706:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013710:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234040.013711:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234040.013714:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234040.013716:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013716:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234040.013717:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013719:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234040.013720:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234040.013721:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234040.013723:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234040.013724:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234040.013726:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234040.013730:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226624:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234040.013732:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013733:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234040.013734:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002b682c08 x1846784245226560/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234040.013738:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234040.013740:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234040.013741:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234040.013743:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234040.013744:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234040.013745:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234040.013747:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234040.013749:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:3.0:1761234040.013752:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234040.013753:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234040.013754:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002b682c08. 00000100:00000001:3.0:1761234040.013757:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234040.013758:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013759:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234040.013760:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000bb0f086 x1846784245226624/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234040.013764:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234040.013765:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234040.013765:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234040.013768:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234040.013770:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234040.013770:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234040.013772:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234040.013772:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:3.0:1761234040.013774:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234040.013774:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234040.013775:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000bb0f086. 00000100:00000001:3.0:1761234040.013776:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234040.013777:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013778:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013785:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234040.013786:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013787:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234040.013788:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234040.013789:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234040.013790:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013794:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234040.013794:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234040.013795:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234040.013797:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234040.013798:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226688:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234040.013800:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234040.013801:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234040.013802:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234040.013803:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234040.013804:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234040.013805:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234040.013806:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234040.013808:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234040.013809:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226688, portal 4 00000100:00000001:3.0:1761234040.013810:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234040.013811:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039814912 : -104717669736704 : ffffa0c2847e5f00) 00000100:00000040:3.0:1761234040.013814:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234065 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013818:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234040.013820:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234040.013821:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234040.013822:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245226688, offset 0 00000400:00000200:3.0:1761234040.013825:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234040.013827:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234040.013829:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234040.013830:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234040.013831:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234040.013833:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11759:11759:256:4294967295] 192.168.201.124@tcp LPNI seq info [11759:11759:8:4294967295] 00000400:00000200:3.0:1761234040.013838:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234040.013865:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234040.013867:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234040.013870:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234040.013872:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1761234040.013874:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234040.013876:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1761234040.013878:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234040.013880:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234040.013882:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234040.013884:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234040.013885:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234040.013886:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234040.013888:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234040.013889:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234065 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013894:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234040.013895:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234065 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013899:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013900:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234040.013903:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013904:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234040.013905:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013906:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234040.013908:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234040/real 1761234040] req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234040.013912:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234040.013913:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234040.013913:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234040.013914:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234040.013915:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234040.013916:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234040.013918:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234040.013922:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013925:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234040.013926:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013927:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234040.013927:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234040.013929:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234040.013932:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234040.013933:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234040.013937:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234040.013939:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013940:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234040.013941:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013942:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234040.013943:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234040.013944:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234040.013946:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234040.013947:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234040.013949:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234040.013953:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226688:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234040.013955:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013956:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234040.013957:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000632be758 x1846784245226688/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234065 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234040.013960:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234040.013961:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234040.013962:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234040.013964:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234040.013965:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234040.013966:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234040.013967:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234040.013968:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a174ebf2. 02000000:00000001:3.0:1761234040.013973:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234040.013974:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234040.013975:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000632be758. 00000100:00000001:3.0:1761234040.013978:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234040.013979:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013980:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234040.013983:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234040.013984:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234040.013986:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234040.013987:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234040.524957:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234040.524961:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234040.524962:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234040.524966:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234040.524970:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234040.524972:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234040.524974:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234040.524976:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234040.524978:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234040.908922:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234040.908926:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234040.908932:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234040.908933:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234040.908935:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234040.908937:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234040.909045:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234040.909050:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234040.909054:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234041.548959:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234041.548963:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234041.548965:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234041.548968:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234041.548972:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234041.548975:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234041.548977:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234041.548978:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234041.548980:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234041.932935:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234041.932941:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234041.932948:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234041.932949:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234041.932952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234041.932953:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234041.933044:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234041.933048:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234041.933050:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234042.573000:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234042.573007:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234042.573008:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234042.573013:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234042.573020:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234042.573024:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234042.573028:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234042.573030:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234042.573033:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234042.956923:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234042.956929:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234042.956936:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234042.956937:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234042.956940:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234042.956942:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234042.957028:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234042.957032:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234042.957035:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234043.597013:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234043.597017:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234043.597019:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234043.597023:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234043.597027:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234043.597030:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234043.597033:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234043.597035:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234043.597037:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234043.980968:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234043.980972:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234043.980977:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234043.980979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234043.980982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234043.980983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234043.981044:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234043.981048:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234043.981050:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234044.620980:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234044.620985:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234044.620986:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234044.620989:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234044.620993:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234044.620996:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234044.620998:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234044.620999:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234044.621001:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234045.004910:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234045.004915:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234045.004920:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234045.004921:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234045.004924:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234045.004925:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234045.004992:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234045.004995:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234045.004999:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761234045.133007:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234045.133017:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234045.133018:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:2.0:1761234045.133021:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234045.133023:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234045.133027:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234045.133032:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6514 00000100:00080000:2.0:1761234045.133035:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234045.133037:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133039:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234045.133041:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133042:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234045.133045:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234045.133047:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:2.0:1761234045.133049:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234045.133051:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:2.0:1761234045.133052:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:2.0:1761234045.133054:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:2.0:1761234045.133055:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133056:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234045.133057:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133058:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234045.133060:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:2.0:1761234045.133062:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133064:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234045.133067:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234045.133069:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234045.133071:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133072:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234045.133073:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234045.133081:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e818ed53. 00000020:00000040:2.0:1761234045.133084:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:2.0:1761234045.133087:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234045.133088:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133090:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234045.133092:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234045.133094:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234045.133100:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000003562879b. 00000100:00000001:2.0:1761234045.133105:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234045.133110:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234045.133117:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234045.133119:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234045.133122:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e818ed53] to pc [ptlrpcd_rcv+-1] req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234045.133139:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133141:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234045.133143:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234045.133146:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234045.133147:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1761234045.133148:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234045.133150:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234045.133152:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234045.133154:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6514 00000100:00080000:2.0:1761234045.133156:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234045.133170:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133173:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234045.133174:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133176:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234045.133178:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234045.133179:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:2.0:1761234045.133180:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234045.133182:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:2.0:1761234045.133185:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:2.0:1761234045.133186:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:2.0:1761234045.133189:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133190:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234045.133192:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133194:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234045.133195:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00000001:3.0:1761234045.133197:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1761234045.133198:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133200:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133200:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1761234045.133202:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1761234045.133203:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1761234045.133207:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1761234045.133207:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234045.133209:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133211:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234045.133213:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1761234045.133215:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1761234045.133216:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133217:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133218:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234045.133220:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234045.133223:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234045.133226:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000014a21c5a. 00000100:00100000:3.0:1761234045.133226:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e818ed53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226752:192.168.201.124@tcp:250: 00000020:00000040:2.0:1761234045.133228:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:2.0:1761234045.133230:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:3.0:1761234045.133230:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234045.133231:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133232:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234045.133233:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234045.133234:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234045.133235:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234045.133236:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234045.133238:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000070e0ffb1. 02000000:00000001:3.0:1761234045.133239:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1761234045.133241:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234045.133242:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 00000100:00100000:2.0:1761234045.133244:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234045.133244:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133250:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1761234045.133250:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00000001:2.0:1761234045.133251:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234045.133254:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000014a21c5a] to pc [ptlrpcd_rcv+-1] req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:3.0:1761234045.133256:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000001:2.0:1761234045.133260:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133262:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000200:3.0:1761234045.133262:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226752, portal 25 00000100:00080000:2.0:1761234045.133263:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234045.133264:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234045.133266:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090815232 : -104717618736384 : ffffa0c287889300) 00000100:00000001:2.0:1761234045.133267:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234045.133267:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1761234045.133269:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234045.133270:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1761234045.133270:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234070 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234045.133273:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234045.133275:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6514 00000100:00000001:3.0:1761234045.133276:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00080000:2.0:1761234045.133277:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234045.133279:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000040:3.0:1761234045.133280:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133282:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1761234045.133282:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000040:2.0:1761234045.133283:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000200:3.0:1761234045.133284:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245226752, offset 0 00000100:00000001:2.0:1761234045.133285:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234045.133287:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234045.133288:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:2.0:1761234045.133289:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234045.133291:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000400:00000200:3.0:1761234045.133291:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000020:00000001:2.0:1761234045.133293:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:2.0:1761234045.133294:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234045.133296:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133297:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234045.133299:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133300:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234045.133302:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:2.0:1761234045.133304:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000200:3.0:1761234045.133304:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:2.0:1761234045.133306:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234045.133308:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234045.133309:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234045.133311:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234045.133311:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000001:2.0:1761234045.133313:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234045.133314:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1761234045.133315:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000008:00000001:2.0:1761234045.133316:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133318:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234045.133318:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000010:2.0:1761234045.133321:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcc2ec0d. 00000020:00000040:2.0:1761234045.133322:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000200:3.0:1761234045.133322:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11760:11760:256:4294967295] 192.168.201.124@tcp LPNI seq info [11760:11760:8:4294967295] 02000000:00000001:2.0:1761234045.133325:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234045.133326:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133327:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234045.133328:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234045.133329:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234045.133332:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000dd1dcb0e. 00000400:00000200:3.0:1761234045.133332:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:2.0:1761234045.133334:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234045.133337:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.0:1761234045.133339:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000001:2.0:1761234045.133344:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000800:00000010:3.0:1761234045.133344:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000100:00000001:2.0:1761234045.133345:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234045.133347:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcc2ec0d] to pc [ptlrpcd_rcv+-1] req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.0:1761234045.133350:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234045.133353:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1761234045.133354:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234045.133356:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:2.0:1761234045.133359:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6524) 00000800:00000200:3.1:1761234045.133360:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234045.133362:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1761234045.133366:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234045.133369:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234045.133375:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234045.133380:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234045.133384:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234045.133386:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234045.133390:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234045.133393:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234070 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133402:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234045.133404:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234070 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133410:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133411:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234045.133417:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133418:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234045.133420:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133422:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234045.133424:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234045/real 1761234045] req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234045.133431:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234045.133432:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234045.133434:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234045.133435:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234045.133436:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234045.133438:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234045.133439:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234045.133444:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133448:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234045.133449:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133450:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234045.133451:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234045.133453:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133456:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234045.133459:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234045.133463:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234045.133465:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133466:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234045.133467:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133468:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234045.133469:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234045.133471:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234045.133473:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234045.133474:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234045.133476:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234045.133480:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e818ed53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226752:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234045.133483:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133484:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234045.133485:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000e818ed53 x1846784245226752/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234045.133489:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234045.133491:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234045.133493:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234045.133496:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234045.133497:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234045.133498:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234045.133499:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234045.133501:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000003562879b. 02000000:00000001:3.0:1761234045.133503:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234045.133505:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234045.133505:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000e818ed53. 00000100:00000001:3.0:1761234045.133508:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234045.133508:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133510:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133516:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234045.133517:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133518:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234045.133519:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234045.133520:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234045.133521:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133525:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234045.133526:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234045.133527:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234045.133528:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234045.133530:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000014a21c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226816:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234045.133532:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234045.133533:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234045.133534:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234045.133535:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234045.133536:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234045.133537:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234045.133539:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234045.133541:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234045.133543:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226816, portal 10 00000100:00000001:3.0:1761234045.133544:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234045.133545:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090838336 : -104717618713280 : ffffa0c28788ed40) 00000100:00000040:3.0:1761234045.133549:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234070 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133552:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234045.133554:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234045.133555:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234045.133556:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245226816, offset 0 00000400:00000200:3.0:1761234045.133559:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234045.133563:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234045.133565:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234045.133567:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234045.133568:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234045.133570:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11761:11761:256:4294967295] 192.168.201.124@tcp LPNI seq info [11761:11761:8:4294967295] 00000400:00000200:3.0:1761234045.133575:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234045.133578:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234045.133579:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234045.133582:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234045.133583:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1761234045.133586:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234045.133588:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1761234045.133590:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234045.133592:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234045.133593:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234045.133596:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234045.133597:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234045.133598:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234045.133600:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234045.133601:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234070 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133605:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234045.133607:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234070 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133614:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133615:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234045.133617:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133618:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234045.133619:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133620:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234045.133621:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234045/real 1761234045] req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234045.133626:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234045.133627:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234045.133627:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234045.133628:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234045.133629:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234045.133630:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234045.133631:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234045.133635:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133638:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234045.133639:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133639:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234045.133640:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234045.133642:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133645:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234045.133646:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234045.133650:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234045.133651:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133652:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234045.133653:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133654:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234045.133655:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234045.133657:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234045.133658:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234045.133660:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234045.133661:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234045.133665:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000014a21c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226816:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234045.133667:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234045.133668:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133671:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234045.133672:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234045.133673:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234045.133674:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234045.133675:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcc2ec0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226880:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234045.133677:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234045.133678:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234045.133679:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234045.133680:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234045.133681:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234045.133682:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234045.133684:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234045.133685:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234045.133687:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226880, portal 4 00000100:00000001:3.0:1761234045.133688:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234045.133688:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090829824 : -104717618721792 : ffffa0c28788cc00) 00000100:00000040:3.0:1761234045.133691:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234070 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133695:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234045.133697:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234045.133698:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234045.133699:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245226880, offset 0 00000400:00000200:3.0:1761234045.133701:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234045.133704:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234045.133706:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234045.133707:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234045.133708:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234045.133710:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11762:11762:256:4294967295] 192.168.201.124@tcp LPNI seq info [11762:11762:8:4294967295] 00000400:00000200:3.0:1761234045.133715:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234045.133717:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234045.133719:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234045.133721:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234045.133722:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1761234045.133724:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234045.133725:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1761234045.133728:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234045.133729:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234045.133730:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234045.133732:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234045.133734:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234045.133735:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234045.133736:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234045.133737:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234070 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133741:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234045.133742:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234070 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133745:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133746:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234045.133747:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133748:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234045.133749:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133750:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234045.133752:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234045/real 1761234045] req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234045.133756:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234045.133757:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234045.133758:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234045.133759:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234045.133759:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234045.133760:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234045.133761:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234045.133765:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133768:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234045.133769:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133770:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234045.133770:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234045.133772:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234045.133775:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234045.133776:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234045.133780:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234045.133781:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133782:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234045.133782:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133783:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234045.133784:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234045.133785:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234045.133786:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234045.133788:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234045.133790:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234045.133794:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcc2ec0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245226880:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234045.133796:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133797:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234045.133798:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000014a21c5a x1846784245226816/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234045.133801:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234045.133802:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234045.133803:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234045.133804:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234045.133804:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234045.133805:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234045.133806:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234045.133807:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000070e0ffb1. 02000000:00000001:3.0:1761234045.133809:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234045.133810:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234045.133810:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000014a21c5a. 00000100:00000001:3.0:1761234045.133814:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234045.133814:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133815:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234045.133817:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fcc2ec0d x1846784245226880/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234070 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234045.133820:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234045.133821:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234045.133821:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234045.133822:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234045.133823:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234045.133824:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234045.133824:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234045.133825:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000dd1dcb0e. 02000000:00000001:3.0:1761234045.133826:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234045.133827:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234045.133828:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fcc2ec0d. 00000100:00000001:3.0:1761234045.133832:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234045.133832:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133833:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234045.133835:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234045.133836:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234045.133837:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234045.133838:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234045.645052:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234045.645060:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234045.645062:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234045.645068:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234045.645075:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234045.645080:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234045.645082:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234045.645085:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234045.645088:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234046.028929:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234046.028934:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234046.028939:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234046.028939:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234046.028941:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234046.028942:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234046.028995:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234046.028999:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234046.029001:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234046.669042:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234046.669048:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234046.669050:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234046.669083:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234046.669087:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234046.669090:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234046.669092:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234046.669094:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234046.669095:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234047.052939:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234047.052943:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234047.052949:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234047.052950:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234047.052953:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234047.052953:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234047.053042:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234047.053045:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234047.053048:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234047.692977:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234047.692981:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234047.692983:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234047.692987:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234047.692991:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234047.692994:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234047.692996:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234047.692999:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234047.693001:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234048.076946:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234048.076950:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234048.076955:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234048.076956:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234048.076958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234048.076959:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234048.077024:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234048.077027:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234048.077029:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234048.716960:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234048.716967:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234048.716968:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234048.716973:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234048.716978:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234048.716983:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234048.716985:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234048.716987:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234048.716989:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234049.100977:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234049.100982:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234049.100990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234049.100991:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234049.100995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234049.100996:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234049.101055:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234049.101058:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234049.101060:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234049.740999:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234049.741003:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234049.741005:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234049.741009:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234049.741014:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234049.741017:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234049.741019:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234049.741021:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234049.741023:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.124948:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234050.124954:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234050.124957:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234050.124959:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.124972:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234050.124974:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.124975:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.124976:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.124982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234050.124983:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234050.124984:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234050.124987:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234050.124989:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234050.124990:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.124991:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234050.124991:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.124992:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.124999:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.125002:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234050.125002:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234050.125003:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234050.125005:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234050.125005:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234050.125006:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.125007:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234050.125008:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.125008:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234050.125010:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234050.125087:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234050.125090:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234050.125093:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234050.252982:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234050.252992:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234050.252993:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234050.252996:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234050.252998:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234050.253002:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234050.253008:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6519 00000100:00080000:1.0:1761234050.253011:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234050.253013:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253015:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234050.253016:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253023:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234050.253025:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234050.253027:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234050.253029:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234050.253031:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234050.253033:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234050.253034:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234050.253036:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253037:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234050.253038:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253038:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234050.253040:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234050.253043:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253045:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234050.253047:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234050.253049:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234050.253051:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253052:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234050.253054:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234050.253074:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761234050.253077:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234050.253080:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234050.253081:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253083:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234050.253085:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234050.253088:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234050.253096:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761234050.253101:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234050.253107:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234050.253115:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234050.253117:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234050.253119:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234050.253134:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253136:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234050.253137:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234050.253139:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234050.253140:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234050.253141:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234050.253142:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234050.253144:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234050.253145:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6519 00000100:00080000:1.0:1761234050.253147:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234050.253148:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253149:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234050.253150:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253151:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234050.253152:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234050.253153:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234050.253153:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234050.253154:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234050.253156:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234050.253157:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234050.253158:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253159:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234050.253160:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253161:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234050.253163:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234050.253164:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253166:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234050.253167:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234050.253171:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234050.253172:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253173:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234050.253176:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234050.253178:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253180:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234050.253182:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761234050.253183:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234050.253185:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234050.253185:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253187:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234050.253187:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234050.253189:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234050.253191:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234050.253194:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253195:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1761234050.253197:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253199:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234050.253201:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1761234050.253203:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234050.253205:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234050.253209:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:3.0:1761234050.253209:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253237:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234050.253238:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253240:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000001:3.0:1761234050.253241:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761234050.253242:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234050.253245:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234050.253246:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234050.253247:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1761234050.253248:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1761234050.253249:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234050.253252:0:468:0:(import.c:519:import_select_connection()) Process entered 02000000:00000001:3.0:1761234050.253252:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761234050.253253:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6519 00000100:00080000:1.0:1761234050.253256:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1761234050.253260:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226944:192.168.201.124@tcp:250: 00000100:00000040:1.0:1761234050.253261:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253263:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234050.253266:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253269:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234050.253270:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000100:00000001:3.0:1761234050.253270:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1761234050.253272:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234050.253273:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234050.253274:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234050.253276:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:3.0:1761234050.253276:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1761234050.253279:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:3.0:1761234050.253280:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234050.253281:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234050.253284:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234050.253286:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 02000000:00000001:3.0:1761234050.253286:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1761234050.253288:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234050.253290:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234050.253291:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 02000000:00000010:3.0:1761234050.253292:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 00000100:00000001:1.0:1761234050.253293:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234050.253295:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234050.253297:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234050.253298:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234050.253302:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253303:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234050.253304:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234050.253305:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253306:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234050.253306:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00000010:1.0:1761234050.253311:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000020:00000040:1.0:1761234050.253313:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234050.253315:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000400:00000010:3.0:1761234050.253316:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 02000000:00000001:1.0:1761234050.253317:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253319:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234050.253321:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234050.253322:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234050.253325:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000200:3.0:1761234050.253326:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245226944, portal 25 00000100:00000001:1.0:1761234050.253328:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253331:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1761234050.253332:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253333:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039807616 : -104717669744000 : ffffa0c2847e4280) 00000100:00000001:1.0:1761234050.253340:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1761234050.253341:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234075 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234050.253342:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234050.253348:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253357:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:1.0:1761234050.253359:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234050.253363:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:3.0:1761234050.253363:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000040:1.0:1761234050.253367:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6529) 00000400:00000010:3.0:1761234050.253367:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234050.253372:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245226944, offset 0 00000400:00000200:3.0:1761234050.253378:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234050.253391:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234050.253397:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234050.253400:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234050.253402:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234050.253404:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11763:11763:256:4294967295] 192.168.201.124@tcp LPNI seq info [11763:11763:8:4294967295] 00000400:00000200:3.0:1761234050.253412:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234050.253417:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234050.253420:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234050.253425:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234050.253428:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761234050.253432:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234050.253434:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1761234050.253437:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234050.253439:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234050.253444:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234050.253447:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234050.253450:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234050.253451:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234050.253455:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234050.253457:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234075 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253464:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234050.253465:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234075 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253469:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253470:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234050.253475:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253477:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234050.253478:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253480:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234050.253482:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234050/real 1761234050] req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234050.253487:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234050.253489:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234050.253489:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234050.253491:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234050.253491:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234050.253493:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234050.253494:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234050.253499:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253502:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234050.253503:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253505:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234050.253506:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234050.253508:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253511:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234050.253514:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234050.253518:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234050.253519:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253520:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234050.253521:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253522:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234050.253523:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234050.253525:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234050.253527:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234050.253529:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234050.253531:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234050.253535:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245226944:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234050.253538:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253539:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234050.253540:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a014c997 x1846784245226944/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234050.253544:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234050.253545:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234050.253547:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234050.253549:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234050.253550:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234050.253551:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234050.253552:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234050.253553:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a174ebf2. 02000000:00000001:3.0:1761234050.253557:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234050.253558:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234050.253559:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a014c997. 00000100:00000001:3.0:1761234050.253560:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234050.253561:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253563:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253568:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234050.253569:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253570:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234050.253571:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234050.253572:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234050.253574:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253578:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234050.253579:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234050.253580:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234050.253581:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234050.253582:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227008:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234050.253584:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234050.253586:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234050.253586:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234050.253588:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234050.253589:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234050.253590:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234050.253592:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234050.253594:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234050.253595:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227008, portal 10 00000100:00000001:3.0:1761234050.253597:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234050.253597:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039812480 : -104717669739136 : ffffa0c2847e5580) 00000100:00000040:3.0:1761234050.253601:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234075 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253605:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234050.253606:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234050.253608:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234050.253609:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245227008, offset 0 00000400:00000200:3.0:1761234050.253611:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234050.253614:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234050.253616:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234050.253617:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234050.253618:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234050.253620:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11764:11764:256:4294967295] 192.168.201.124@tcp LPNI seq info [11764:11764:8:4294967295] 00000400:00000200:3.0:1761234050.253625:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234050.253628:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234050.253630:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234050.253632:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234050.253633:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761234050.253635:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234050.253637:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1761234050.253639:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234050.253640:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234050.253642:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234050.253644:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234050.253646:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234050.253646:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234050.253647:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234050.253649:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234075 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253653:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234050.253654:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234075 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253657:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253658:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234050.253660:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253661:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234050.253662:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253662:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234050.253664:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234050/real 1761234050] req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234050.253668:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234050.253669:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234050.253670:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234050.253671:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234050.253672:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234050.253672:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234050.253673:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234050.253677:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253680:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234050.253681:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253682:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234050.253683:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234050.253684:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253688:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234050.253690:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234050.253694:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234050.253695:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253696:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234050.253696:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253697:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234050.253698:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234050.253699:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234050.253700:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234050.253701:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234050.253703:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234050.253707:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227008:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234050.253708:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234050.253710:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253713:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234050.253714:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234050.253715:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234050.253716:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234050.253717:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227072:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234050.253719:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234050.253720:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234050.253720:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234050.253721:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234050.253723:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234050.253724:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234050.253725:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234050.253727:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234050.253728:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227072, portal 4 00000100:00000001:3.0:1761234050.253729:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234050.253730:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039808832 : -104717669742784 : ffffa0c2847e4740) 00000100:00000040:3.0:1761234050.253732:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234075 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253736:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234050.253738:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234050.253739:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234050.253740:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245227072, offset 0 00000400:00000200:3.0:1761234050.253742:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234050.253744:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234050.253746:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234050.253747:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234050.253749:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234050.253750:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11765:11765:256:4294967295] 192.168.201.124@tcp LPNI seq info [11765:11765:8:4294967295] 00000400:00000200:3.0:1761234050.253755:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234050.253757:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234050.253758:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234050.253760:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234050.253761:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761234050.253763:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234050.253764:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1761234050.253767:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234050.253768:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234050.253769:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234050.253771:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234050.253772:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234050.253773:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234050.253774:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234050.253775:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234075 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253780:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234050.253781:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234075 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253785:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253786:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234050.253787:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253788:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234050.253789:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253789:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234050.253791:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234050/real 1761234050] req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234050.253794:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234050.253795:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234050.253796:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234050.253796:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234050.253797:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234050.253798:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234050.253800:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234050.253806:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253811:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234050.253813:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253814:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234050.253815:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234050.253818:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234050.253824:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234050.253826:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234050.253832:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234050.253835:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253836:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234050.253837:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253839:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234050.253840:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234050.253841:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234050.253870:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234050.253872:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234050.253876:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234050.253883:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227072:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234050.253887:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253888:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234050.253890:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004567def3 x1846784245227008/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234050.253896:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234050.253898:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234050.253899:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234050.253901:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234050.253903:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234050.253903:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234050.253906:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234050.253907:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:3.0:1761234050.253909:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234050.253910:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234050.253911:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004567def3. 00000100:00000001:3.0:1761234050.253914:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234050.253915:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253916:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234050.253918:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f33bf85c x1846784245227072/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234075 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234050.253925:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234050.253926:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234050.253928:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234050.253930:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234050.253932:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234050.253933:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234050.253934:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234050.253936:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:3.0:1761234050.253937:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234050.253938:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234050.253939:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f33bf85c. 00000100:00000001:3.0:1761234050.253941:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234050.253942:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253943:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234050.253946:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234050.253947:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234050.253948:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234050.253949:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234050.764949:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234050.764954:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234050.764955:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234050.764959:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234050.764964:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234050.764968:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234050.764971:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234050.764973:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234050.764975:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234051.148985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234051.148991:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234051.148998:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234051.148999:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234051.149002:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234051.149003:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234051.149090:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234051.149093:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234051.149097:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234051.788978:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234051.788983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234051.788984:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234051.788988:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234051.788993:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234051.788997:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234051.789014:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234051.789016:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234051.789018:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234052.172983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234052.172989:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234052.172996:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234052.172997:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234052.173000:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234052.173001:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234052.173076:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234052.173081:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234052.173085:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234052.812939:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234052.812943:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234052.812944:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234052.812948:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234052.812952:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234052.812955:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234052.812956:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234052.812958:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234052.812960:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234053.196990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234053.196996:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234053.197002:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234053.197003:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234053.197022:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234053.197023:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234053.197093:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234053.197096:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234053.197099:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234053.836990:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234053.837020:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234053.837021:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234053.837025:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234053.837029:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234053.837032:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234053.837034:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234053.837036:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234053.837038:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234054.220972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234054.220978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234054.220997:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234054.220999:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234054.221003:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234054.221004:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234054.221074:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234054.221078:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234054.221081:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234054.860924:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234054.860928:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234054.860930:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234054.860934:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234054.860939:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234054.860941:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234054.860944:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234054.860945:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234054.860947:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234055.245006:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234055.245011:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234055.245018:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234055.245019:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234055.245025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234055.245027:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234055.245101:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234055.245105:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234055.245108:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234055.372996:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234055.373009:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234055.373011:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234055.373026:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234055.373029:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234055.373034:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234055.373041:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6524 00000100:00080000:1.0:1761234055.373044:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234055.373046:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373049:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234055.373051:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373052:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234055.373057:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234055.373059:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234055.373062:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234055.373065:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234055.373067:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234055.373069:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234055.373071:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373071:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234055.373072:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373073:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234055.373075:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234055.373078:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373079:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234055.373082:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234055.373085:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234055.373087:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373089:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234055.373091:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234055.373101:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761234055.373104:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234055.373109:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234055.373111:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373114:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234055.373116:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234055.373120:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234055.373128:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761234055.373135:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234055.373142:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234055.373150:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234055.373152:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234055.373155:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234055.373176:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373177:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234055.373179:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234055.373182:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234055.373182:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234055.373183:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234055.373184:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234055.373185:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234055.373187:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6524 00000100:00080000:1.0:1761234055.373188:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234055.373190:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373191:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234055.373191:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373193:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234055.373194:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234055.373195:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234055.373196:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234055.373198:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234055.373199:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234055.373200:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234055.373202:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373202:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234055.373204:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373205:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234055.373206:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234055.373207:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373208:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234055.373210:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234055.373216:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234055.373217:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373219:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234055.373222:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234055.373225:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373226:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234055.373229:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000100:00000001:3.0:1761234055.373229:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1761234055.373232:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234055.373235:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:3.0:1761234055.373235:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1761234055.373237:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373239:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1761234055.373239:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234055.373242:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234055.373243:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234055.373246:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000040:3.0:1761234055.373247:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234055.373248:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234055.373255:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373261:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234055.373263:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234055.373264:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373265:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234055.373268:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1761234055.373269:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234055.373272:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373278:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234055.373278:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227136:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761234055.373280:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234055.373282:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234055.373284:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1761234055.373285:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234055.373286:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234055.373288:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234055.373289:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1761234055.373290:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1761234055.373291:0:468:0:(import.c:519:import_select_connection()) Process entered 02000000:00000001:3.0:1761234055.373292:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761234055.373295:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6524 02000000:00000001:3.0:1761234055.373296:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1761234055.373299:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234055.373301:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373303:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000010:3.0:1761234055.373303:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234055.373306:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234055.373307:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373310:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234055.373312:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234055.373313:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234055.373313:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000400:00000010:3.0:1761234055.373314:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000020:00000040:1.0:1761234055.373315:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234055.373318:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234055.373320:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234055.373321:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373322:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1761234055.373323:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000040:1.0:1761234055.373324:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373327:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234055.373328:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234055.373330:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373331:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234055.373332:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:3.0:1761234055.373333:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227136, portal 25 02000000:00000001:1.0:1761234055.373334:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234055.373338:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373338:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761234055.373341:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234055.373341:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039795456 : -104717669756160 : ffffa0c2847e1300) 00000008:00100000:1.0:1761234055.373343:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234055.373346:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373347:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234055.373349:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234080 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000010:1.0:1761234055.373351:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761234055.373354:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234055.373357:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234055.373358:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373360:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1761234055.373362:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 02000000:00000001:1.0:1761234055.373363:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234055.373364:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234055.373368:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000040:3.0:1761234055.373368:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234055.373371:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234055.373372:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00100000:1.0:1761234055.373376:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234055.373376:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245227136, offset 0 00000100:00000001:1.0:1761234055.373382:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234055.373383:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234055.373383:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000040:1.0:1761234055.373387:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234055.373392:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234055.373393:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234055.373396:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6534) 00000400:00000200:3.0:1761234055.373402:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234055.373411:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234055.373415:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234055.373418:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234055.373422:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11766:11766:256:4294967295] 192.168.201.124@tcp LPNI seq info [11766:11766:8:4294967295] 00000400:00000200:3.0:1761234055.373431:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234055.373436:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234055.373440:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234055.373447:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234055.373450:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761234055.373455:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234055.373457:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1761234055.373460:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234055.373463:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234055.373469:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234055.373472:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234055.373477:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234055.373479:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234055.373493:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234055.373496:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234080 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373503:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234055.373505:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234080 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373509:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373510:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234055.373516:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373518:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234055.373520:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373522:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234055.373524:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234055/real 1761234055] req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234055.373531:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234055.373532:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234055.373533:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234055.373534:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234055.373535:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234055.373540:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234055.373542:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234055.373547:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373551:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234055.373552:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373554:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234055.373555:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234055.373557:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373561:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234055.373564:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234055.373573:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234055.373575:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373576:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234055.373576:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373578:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234055.373579:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234055.373581:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234055.373583:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234055.373584:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234055.373587:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234055.373591:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227136:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234055.373594:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373595:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234055.373596:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fd54b549 x1846784245227136/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234055.373600:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234055.373602:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234055.373604:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234055.373607:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234055.373608:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234055.373609:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234055.373610:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234055.373612:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006d8d01a9. 02000000:00000001:3.0:1761234055.373615:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234055.373616:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234055.373617:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fd54b549. 00000100:00000001:3.0:1761234055.373620:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234055.373620:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373622:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373629:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234055.373631:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373633:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234055.373633:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234055.373634:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234055.373636:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373640:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234055.373641:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234055.373642:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234055.373643:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234055.373645:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000643267a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227200:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234055.373647:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234055.373648:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234055.373648:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234055.373649:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234055.373651:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234055.373652:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234055.373654:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234055.373655:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234055.373657:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227200, portal 10 00000100:00000001:3.0:1761234055.373658:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234055.373659:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039800320 : -104717669751296 : ffffa0c2847e2600) 00000100:00000040:3.0:1761234055.373664:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234080 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373668:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234055.373670:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234055.373671:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234055.373672:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245227200, offset 0 00000400:00000200:3.0:1761234055.373675:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234055.373677:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234055.373679:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234055.373681:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234055.373682:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234055.373684:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11767:11767:256:4294967295] 192.168.201.124@tcp LPNI seq info [11767:11767:8:4294967295] 00000400:00000200:3.0:1761234055.373689:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234055.373692:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234055.373694:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234055.373696:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234055.373698:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761234055.373701:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234055.373702:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1761234055.373704:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234055.373705:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234055.373707:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234055.373709:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234055.373710:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234055.373712:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234055.373713:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234055.373714:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234080 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373718:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234055.373719:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234080 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373723:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373724:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234055.373726:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373727:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234055.373728:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373733:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234055.373735:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234055/real 1761234055] req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234055.373739:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234055.373740:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234055.373741:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234055.373742:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234055.373742:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234055.373744:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234055.373745:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234055.373750:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373754:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234055.373755:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373756:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234055.373756:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234055.373758:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373761:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234055.373762:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234055.373766:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234055.373767:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373768:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234055.373768:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373769:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234055.373770:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234055.373771:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234055.373772:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234055.373774:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234055.373776:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234055.373780:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000643267a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227200:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234055.373782:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234055.373783:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373787:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234055.373788:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234055.373789:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234055.373790:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234055.373791:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d6a5077 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227264:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234055.373794:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234055.373794:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234055.373795:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234055.373796:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234055.373798:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234055.373799:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234055.373800:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234055.373802:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234055.373804:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227264, portal 4 00000100:00000001:3.0:1761234055.373805:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234055.373806:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039794240 : -104717669757376 : ffffa0c2847e0e40) 00000100:00000040:3.0:1761234055.373808:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234080 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373812:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234055.373814:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234055.373815:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234055.373816:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245227264, offset 0 00000400:00000200:3.0:1761234055.373818:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234055.373821:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234055.373823:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234055.373824:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234055.373826:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234055.373828:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11768:11768:256:4294967295] 192.168.201.124@tcp LPNI seq info [11768:11768:8:4294967295] 00000400:00000200:3.0:1761234055.373833:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234055.373835:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234055.373836:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234055.373838:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234055.373840:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761234055.373865:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234055.373867:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1761234055.373869:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234055.373871:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234055.373873:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234055.373875:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234055.373876:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234055.373877:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234055.373878:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234055.373880:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234080 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373885:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234055.373886:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234080 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373890:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373891:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234055.373893:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373894:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234055.373895:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373897:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234055.373898:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234055/real 1761234055] req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234055.373903:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234055.373903:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234055.373904:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234055.373905:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234055.373906:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234055.373907:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234055.373908:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234055.373912:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373916:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234055.373916:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373918:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234055.373919:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234055.373920:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234055.373925:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234055.373926:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234055.373929:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234055.373931:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373931:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234055.373932:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373933:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234055.373933:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234055.373935:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234055.373936:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234055.373937:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234055.373938:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234055.373942:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d6a5077 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227264:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234055.373944:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373945:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234055.373947:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000643267a x1846784245227200/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234055.373951:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234055.373951:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234055.373952:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234055.373954:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234055.373954:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234055.373955:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234055.373956:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234055.373957:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:3.0:1761234055.373959:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234055.373959:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234055.373960:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000643267a. 00000100:00000001:3.0:1761234055.373962:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234055.373962:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373975:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234055.373977:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007d6a5077 x1846784245227264/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234080 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234055.373981:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234055.373982:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234055.373983:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234055.373985:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234055.373985:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234055.373986:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234055.373987:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234055.373988:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:3.0:1761234055.373990:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234055.373990:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234055.373991:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007d6a5077. 00000100:00000001:3.0:1761234055.373992:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234055.373993:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373993:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234055.373995:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234055.373996:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234055.373997:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234055.373998:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234055.884942:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234055.884946:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234055.884948:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234055.884951:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234055.884981:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234055.884984:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234055.884986:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234055.884988:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234055.884990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234056.268926:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234056.268930:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234056.268936:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234056.268937:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234056.268939:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234056.268940:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234056.269003:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234056.269006:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234056.269009:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234056.908920:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234056.908925:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234056.908926:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234056.908929:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234056.908933:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234056.908948:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234056.908950:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234056.908952:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234056.908953:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234057.292916:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234057.292921:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234057.292926:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234057.292953:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234057.292956:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234057.292957:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234057.293010:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234057.293013:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234057.293016:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234057.933005:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234057.933010:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234057.933012:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234057.933015:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234057.933020:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234057.933022:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234057.933024:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234057.933026:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234057.933028:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234058.316965:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234058.316970:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234058.316976:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234058.316977:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234058.316980:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234058.316981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234058.317054:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234058.317058:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234058.317061:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234058.956969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234058.956974:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234058.956975:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234058.956979:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234058.956983:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234058.956986:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234058.956988:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234058.956990:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234058.956991:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234059.340985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234059.340988:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234059.340997:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234059.340999:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234059.341001:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234059.341002:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234059.341163:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234059.341166:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234059.341169:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234059.980999:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234059.981004:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234059.981005:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234059.981008:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234059.981012:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234059.981015:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234059.981017:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234059.981019:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234059.981021:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.364963:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234060.364969:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234060.364971:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234060.364973:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.364978:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234060.364980:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.364982:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.364983:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.364986:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234060.364988:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234060.364988:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234060.364992:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234060.364993:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234060.364994:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.364995:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234060.364996:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.364997:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.364998:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.365000:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234060.365001:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234060.365001:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234060.365003:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234060.365003:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234060.365004:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.365005:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234060.365005:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.365006:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234060.365007:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234060.365067:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234060.365071:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234060.365073:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234060.492979:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234060.492988:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234060.492990:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234060.492992:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234060.492994:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234060.492998:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234060.493004:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6529 00000100:00080000:1.0:1761234060.493006:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234060.493009:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493011:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234060.493012:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493014:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234060.493017:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234060.493018:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234060.493020:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234060.493023:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234060.493025:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234060.493026:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234060.493028:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493029:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234060.493030:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493031:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234060.493033:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234060.493035:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493036:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234060.493038:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234060.493040:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234060.493042:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234060.493043:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234060.493045:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234060.493052:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761234060.493054:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234060.493057:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234060.493058:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234060.493060:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234060.493062:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234060.493065:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234060.493071:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761234060.493076:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234060.493081:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234060.493087:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234060.493089:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234060.493092:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234060.493111:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234060.493113:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234060.493115:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234060.493118:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234060.493119:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234060.493121:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234060.493122:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234060.493124:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234060.493126:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6529 00000100:00080000:1.0:1761234060.493128:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234060.493131:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493132:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234060.493134:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493135:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234060.493137:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234060.493138:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234060.493139:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234060.493141:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234060.493142:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234060.493144:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234060.493146:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493148:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234060.493150:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493151:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234060.493153:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234060.493156:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493158:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234060.493160:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234060.493164:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234060.493166:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493166:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234060.493168:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234060.493170:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1761234060.493170:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234060.493172:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000008:00000001:1.0:1761234060.493174:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234060.493175:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234060.493176:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000010:1.0:1761234060.493178:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761234060.493180:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234060.493183:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234060.493184:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234060.493186:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234060.493187:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234060.493188:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493191:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234060.493192:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234060.493193:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 02000000:00000001:3.0:1761234060.493195:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1761234060.493196:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234060.493197:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234060.493199:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761234060.493200:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005940a194 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227328:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234060.493204:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1761234060.493205:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234060.493206:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234060.493208:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1761234060.493209:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234060.493210:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234060.493212:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1761234060.493216:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234060.493216:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 00000100:00000001:1.0:1761234060.493218:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 02000000:00000001:3.0:1761234060.493218:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761234060.493219:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234060.493222:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234060.493223:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000400:00000010:3.0:1761234060.493223:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00000001:1.0:1761234060.493224:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234060.493225:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234060.493227:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234060.493229:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6529 00000400:00000010:3.0:1761234060.493229:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00080000:1.0:1761234060.493231:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234060.493233:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493235:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1761234060.493235:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227328, portal 25 00000100:00000001:3.0:1761234060.493237:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234060.493239:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039820992 : -104717669730624 : ffffa0c2847e76c0) 00000100:00000040:3.0:1761234060.493243:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234085 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:1.0:1761234060.493245:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493250:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234060.493253:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234060.493254:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234060.493256:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234060.493257:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000001:3.0:1761234060.493257:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000001:1.0:1761234060.493259:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:3.0:1761234060.493260:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000020:00000001:1.0:1761234060.493261:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000010:3.0:1761234060.493262:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000040:1.0:1761234060.493264:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493265:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1761234060.493265:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245227328, offset 0 00000100:00000040:1.0:1761234060.493267:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493268:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234060.493270:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234060.493272:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234060.493274:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234060.493275:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:1.0:1761234060.493276:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234060.493278:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234060.493279:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234060.493281:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234060.493282:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234060.493284:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234060.493285:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234060.493288:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000400:00000200:3.0:1761234060.493288:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000020:00000040:1.0:1761234060.493290:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234060.493292:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234060.493292:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234060.493294:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234060.493295:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234060.493296:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234060.493296:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 02000000:00000010:1.0:1761234060.493299:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000400:00000200:3.0:1761234060.493299:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1761234060.493302:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234060.493302:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00100000:1.0:1761234060.493305:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234060.493306:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11769:11769:256:4294967295] 192.168.201.124@tcp LPNI seq info [11769:11769:8:4294967295] 00000100:00000001:1.0:1761234060.493311:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234060.493313:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234060.493315:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234060.493317:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:1.0:1761234060.493321:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234060.493323:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:3.0:1761234060.493323:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000040:1.0:1761234060.493328:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6539) 00000800:00000010:3.0:1761234060.493328:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234060.493334:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234060.493337:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761234060.493343:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234060.493345:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1761234060.493349:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234060.493351:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234060.493357:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234060.493362:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234060.493367:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234060.493369:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234060.493372:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234060.493376:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234085 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493384:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234060.493386:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234085 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493390:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493391:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234060.493397:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493399:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234060.493400:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493402:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234060.493404:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234060/real 1761234060] req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234060.493411:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234060.493412:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234060.493413:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234060.493414:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234060.493415:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234060.493417:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234060.493418:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234060.493423:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493427:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234060.493428:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493430:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234060.493431:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234060.493433:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493437:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234060.493439:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234060.493443:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234060.493444:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493445:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234060.493446:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493447:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234060.493448:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234060.493450:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234060.493452:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234060.493453:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234060.493455:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234060.493460:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005940a194 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227328:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234060.493462:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493464:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234060.493465:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000005940a194 x1846784245227328/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234060.493468:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234060.493470:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234060.493471:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234060.493473:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234060.493474:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234060.493475:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234060.493476:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234060.493477:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a3ed3058. 02000000:00000001:3.0:1761234060.493480:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234060.493481:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234060.493482:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000005940a194. 00000100:00000001:3.0:1761234060.493484:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234060.493485:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493486:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493493:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234060.493494:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493495:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234060.493496:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234060.493496:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234060.493498:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493502:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234060.493503:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234060.493504:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234060.493505:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234060.493506:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3b47c4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227392:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234060.493508:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234060.493509:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234060.493510:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234060.493511:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234060.493513:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234060.493513:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234060.493515:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234060.493517:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234060.493519:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227392, portal 10 00000100:00000001:3.0:1761234060.493520:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234060.493520:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039818560 : -104717669733056 : ffffa0c2847e6d40) 00000100:00000040:3.0:1761234060.493524:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234085 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493527:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234060.493529:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234060.493531:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234060.493532:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245227392, offset 0 00000400:00000200:3.0:1761234060.493535:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234060.493538:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234060.493541:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234060.493542:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234060.493544:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234060.493545:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11770:11770:256:4294967295] 192.168.201.124@tcp LPNI seq info [11770:11770:8:4294967295] 00000400:00000200:3.0:1761234060.493551:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234060.493553:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234060.493555:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234060.493558:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234060.493559:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761234060.493561:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234060.493563:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1761234060.493565:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234060.493566:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234060.493568:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234060.493570:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234060.493572:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234060.493573:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234060.493574:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234060.493575:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234085 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493579:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234060.493581:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234085 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493584:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493585:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234060.493587:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493588:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234060.493589:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493590:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234060.493591:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234060/real 1761234060] req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234060.493596:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234060.493597:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234060.493598:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234060.493598:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234060.493599:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234060.493600:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234060.493601:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234060.493605:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493608:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234060.493609:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493609:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234060.493610:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234060.493612:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493615:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234060.493617:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234060.493621:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234060.493622:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493623:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234060.493623:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493624:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234060.493625:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234060.493626:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234060.493627:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234060.493628:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234060.493630:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234060.493638:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3b47c4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227392:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234060.493640:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234060.493641:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493645:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234060.493645:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234060.493646:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234060.493647:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234060.493648:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f4f489ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227456:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234060.493650:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234060.493651:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234060.493651:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234060.493652:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234060.493654:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234060.493655:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234060.493656:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234060.493658:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234060.493659:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227456, portal 4 00000100:00000001:3.0:1761234060.493660:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234060.493661:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039793024 : -104717669758592 : ffffa0c2847e0980) 00000100:00000040:3.0:1761234060.493664:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234085 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493667:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234060.493668:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234060.493669:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234060.493670:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245227456, offset 0 00000400:00000200:3.0:1761234060.493672:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234060.493675:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234060.493676:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234060.493678:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234060.493680:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234060.493681:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11771:11771:256:4294967295] 192.168.201.124@tcp LPNI seq info [11771:11771:8:4294967295] 00000400:00000200:3.0:1761234060.493686:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234060.493688:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234060.493690:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234060.493694:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234060.493695:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761234060.493699:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234060.493701:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1761234060.493704:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234060.493706:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234060.493709:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234060.493712:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234060.493714:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234060.493716:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234060.493717:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234060.493720:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234085 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493727:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234060.493729:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234085 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493735:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493737:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234060.493740:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493741:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234060.493743:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493745:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234060.493747:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234060/real 1761234060] req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234060.493755:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234060.493756:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234060.493757:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234060.493759:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234060.493761:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234060.493763:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234060.493764:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234060.493769:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493773:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234060.493773:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493774:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234060.493775:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234060.493776:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234060.493780:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234060.493782:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234060.493786:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234060.493787:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493788:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234060.493788:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493789:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234060.493790:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234060.493791:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234060.493792:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234060.493793:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234060.493795:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234060.493799:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f4f489ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227456:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234060.493801:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493802:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234060.493803:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000c3b47c4a x1846784245227392/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234060.493807:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234060.493808:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234060.493809:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234060.493810:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234060.493811:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234060.493812:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234060.493813:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234060.493814:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:3.0:1761234060.493816:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234060.493816:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234060.493817:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000c3b47c4a. 00000100:00000001:3.0:1761234060.493819:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234060.493819:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493821:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234060.493822:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f4f489ba x1846784245227456/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234085 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234060.493825:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234060.493826:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234060.493826:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234060.493828:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234060.493829:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234060.493829:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234060.493830:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234060.493831:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:3.0:1761234060.493832:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234060.493832:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234060.493833:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f4f489ba. 00000100:00000001:3.0:1761234060.493834:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234060.493835:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493836:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234060.493838:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234060.493839:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234060.493840:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234060.493842:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234061.005046:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234061.005052:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234061.005053:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234061.005058:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234061.005063:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234061.005067:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234061.005069:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234061.005071:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234061.005073:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234061.388944:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234061.388949:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234061.388956:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234061.388957:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234061.388960:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234061.388961:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234061.389057:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234061.389063:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234061.389068:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234062.028989:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234062.028995:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234062.028997:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234062.029001:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234062.029006:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234062.029010:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234062.029012:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234062.029014:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234062.029017:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234062.412934:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234062.412939:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234062.412946:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234062.412947:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234062.412952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234062.412954:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234062.413032:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234062.413037:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234062.413043:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234063.052969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234063.052974:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234063.052976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234063.052979:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234063.052983:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234063.052986:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234063.052988:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234063.052990:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234063.052993:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234063.436953:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234063.436957:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234063.436962:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234063.436963:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234063.436966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234063.436967:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234063.437027:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234063.437030:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234063.437032:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234064.076939:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234064.076944:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234064.076945:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234064.076948:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234064.076953:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234064.076955:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234064.076957:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234064.076959:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234064.076961:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234064.460957:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234064.460962:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234064.460969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234064.460970:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234064.460973:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234064.460974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234064.461018:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234064.461021:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234064.461025:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234065.100967:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234065.100972:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234065.100973:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234065.100978:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234065.100982:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234065.100985:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234065.100986:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234065.100988:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234065.100990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234065.484957:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234065.484964:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234065.484971:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234065.484972:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234065.484976:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234065.484976:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234065.485065:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234065.485068:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234065.485071:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234065.612916:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234065.612925:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234065.612927:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234065.612929:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234065.612931:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234065.612936:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234065.612941:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6534 00000100:00080000:1.0:1761234065.612943:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234065.612946:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.612947:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234065.612949:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.612955:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234065.612958:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234065.612959:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234065.612961:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234065.612964:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234065.612965:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234065.612967:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234065.612969:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.612970:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234065.612971:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.612972:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234065.612973:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234065.612977:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234065.612978:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234065.612980:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234065.612983:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234065.612984:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.612986:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234065.612987:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234065.612994:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761234065.612996:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234065.613000:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234065.613000:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613003:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234065.613005:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234065.613008:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234065.613014:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761234065.613018:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234065.613024:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234065.613031:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234065.613033:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234065.613035:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234065.613052:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613053:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234065.613054:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234065.613057:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234065.613058:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234065.613059:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234065.613060:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234065.613061:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234065.613063:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6534 00000100:00080000:1.0:1761234065.613064:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234065.613065:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.613067:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234065.613068:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.613069:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234065.613071:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234065.613072:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234065.613073:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234065.613074:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234065.613075:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234065.613076:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234065.613077:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.613078:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234065.613079:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.613080:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234065.613081:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234065.613082:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234065.613083:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234065.613084:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234065.613088:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234065.613089:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613090:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234065.613092:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234065.613095:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613096:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234065.613098:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761234065.613099:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234065.613101:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234065.613101:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613102:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234065.613103:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234065.613104:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234065.613106:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761234065.613109:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234065.613111:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613113:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234065.613117:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234065.613118:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761234065.613119:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234065.613122:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613122:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1761234065.613128:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613130:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234065.613131:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1761234065.613132:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234065.613133:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234065.613135:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234065.613136:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234065.613137:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234065.613141:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234065.613144:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6534 00000100:00080000:1.0:1761234065.613147:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1761234065.613147:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1761234065.613149:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234065.613150:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613151:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234065.613154:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.613156:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 02000000:00000001:3.0:1761234065.613156:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1761234065.613159:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234065.613160:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 02000000:00000001:3.0:1761234065.613160:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234065.613161:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234065.613163:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234065.613165:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234065.613167:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00100000:3.0:1761234065.613168:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003630282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227520:192.168.201.124@tcp:250: 00000100:00000040:1.0:1761234065.613170:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234065.613173:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234065.613175:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234065.613177:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1761234065.613178:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234065.613181:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234065.613182:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 02000000:00000001:3.0:1761234065.613182:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1761234065.613185:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234065.613186:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234065.613187:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234065.613189:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234065.613191:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1761234065.613192:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613194:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234065.613195:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234065.613196:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613197:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234065.613198:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 00000100:00000010:1.0:1761234065.613200:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 02000000:00000001:3.0:1761234065.613202:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1761234065.613203:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234065.613206:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234065.613207:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234065.613208:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234065.613208:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234065.613209:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234065.613211:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 02000000:00000010:1.0:1761234065.613212:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234065.613216:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234065.613218:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:3.0:1761234065.613220:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000001:1.0:1761234065.613225:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234065.613226:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234065.613228:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234065.613229:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227520, portal 25 00000100:00000001:3.0:1761234065.613234:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761234065.613236:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613237:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039816128 : -104717669735488 : ffffa0c2847e63c0) 00000100:00000001:1.0:1761234065.613238:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234065.613241:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6544) 00000100:00000040:3.0:1761234065.613246:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234090 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613255:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234065.613261:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234065.613265:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234065.613267:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245227520, offset 0 00000400:00000200:3.0:1761234065.613276:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234065.613289:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234065.613297:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234065.613301:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234065.613304:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234065.613307:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11772:11772:256:4294967295] 192.168.201.124@tcp LPNI seq info [11772:11772:8:4294967295] 00000400:00000200:3.0:1761234065.613317:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234065.613323:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234065.613326:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234065.613333:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234065.613336:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761234065.613342:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234065.613345:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1761234065.613350:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234065.613352:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234065.613358:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234065.613362:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234065.613366:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234065.613368:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234065.613372:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234065.613375:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234090 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613383:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234065.613385:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234090 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613391:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613393:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234065.613399:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613401:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234065.613403:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613406:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234065.613408:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234065/real 1761234065] req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234065.613423:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234065.613425:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234065.613426:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234065.613428:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234065.613429:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234065.613431:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234065.613435:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234065.613441:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613447:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234065.613448:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.613451:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234065.613452:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234065.613455:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613461:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234065.613464:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234065.613470:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234065.613472:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.613474:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234065.613475:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.613477:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234065.613479:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234065.613481:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234065.613484:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234065.613486:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234065.613489:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234065.613502:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003630282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227520:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234065.613507:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.613508:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234065.613510:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000003630282c x1846784245227520/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234065.613517:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234065.613519:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234065.613521:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234065.613526:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234065.613527:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234065.613528:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234065.613531:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234065.613533:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f654abd8. 02000000:00000001:3.0:1761234065.613537:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234065.613539:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234065.613540:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000003630282c. 00000100:00000001:3.0:1761234065.613543:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234065.613544:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.613547:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.613554:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234065.613556:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613558:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234065.613559:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234065.613561:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234065.613563:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613569:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234065.613570:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234065.613572:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234065.613574:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234065.613576:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d3898169 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227584:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234065.613579:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234065.613581:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234065.613582:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234065.613583:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234065.613585:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234065.613587:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234065.613590:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234065.613593:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234065.613596:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227584, portal 10 00000100:00000001:3.0:1761234065.613598:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234065.613599:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039796672 : -104717669754944 : ffffa0c2847e17c0) 00000100:00000040:3.0:1761234065.613603:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234090 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613609:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234065.613612:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234065.613614:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234065.613615:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245227584, offset 0 00000400:00000200:3.0:1761234065.613619:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234065.613624:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234065.613628:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234065.613630:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234065.613632:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234065.613635:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11773:11773:256:4294967295] 192.168.201.124@tcp LPNI seq info [11773:11773:8:4294967295] 00000400:00000200:3.0:1761234065.613642:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234065.613646:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234065.613650:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234065.613654:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234065.613655:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761234065.613659:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234065.613661:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1761234065.613665:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234065.613667:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234065.613669:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234065.613672:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234065.613675:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234065.613677:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234065.613679:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234065.613682:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234090 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613688:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234065.613689:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234090 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613695:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613696:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234065.613699:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613700:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234065.613703:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613704:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234065.613706:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234065/real 1761234065] req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234065.613713:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234065.613714:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234065.613715:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234065.613717:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234065.613718:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234065.613720:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234065.613721:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234065.613728:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613733:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234065.613734:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.613736:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234065.613737:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234065.613739:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613746:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234065.613749:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234065.613754:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234065.613756:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.613758:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234065.613758:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.613760:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234065.613761:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234065.613763:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234065.613765:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234065.613767:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234065.613787:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234065.613794:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d3898169 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227584:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234065.613798:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234065.613800:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613806:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234065.613807:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234065.613809:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234065.613811:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234065.613813:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcae4007 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227648:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234065.613816:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234065.613818:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234065.613819:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234065.613820:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234065.613823:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234065.613825:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234065.613828:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234065.613830:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234065.613833:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227648, portal 4 00000100:00000001:3.0:1761234065.613835:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234065.613836:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039802752 : -104717669748864 : ffffa0c2847e2f80) 00000100:00000040:3.0:1761234065.613840:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234090 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613869:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234065.613872:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234065.613874:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234065.613877:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245227648, offset 0 00000400:00000200:3.0:1761234065.613881:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234065.613886:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234065.613890:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234065.613892:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234065.613894:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234065.613896:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11774:11774:256:4294967295] 192.168.201.124@tcp LPNI seq info [11774:11774:8:4294967295] 00000400:00000200:3.0:1761234065.613904:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234065.613908:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234065.613911:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234065.613915:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234065.613917:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761234065.613921:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234065.613923:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1761234065.613927:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234065.613928:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234065.613931:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234065.613934:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234065.613937:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234065.613938:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234065.613940:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234065.613943:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234090 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613950:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234065.613953:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234090 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613959:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613960:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234065.613963:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613964:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234065.613966:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.613968:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234065.613971:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234065/real 1761234065] req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234065.613978:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234065.613979:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234065.613980:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234065.613982:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234065.613983:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234065.613984:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234065.613986:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234065.613993:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.613998:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234065.613999:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.614001:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234065.614002:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234065.614004:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234065.614010:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234065.614013:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234065.614018:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234065.614020:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.614022:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234065.614023:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.614024:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234065.614026:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234065.614028:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234065.614030:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234065.614032:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234065.614035:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234065.614041:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcae4007 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227648:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234065.614044:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.614046:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234065.614048:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000d3898169 x1846784245227584/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234065.614054:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234065.614056:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234065.614057:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234065.614060:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234065.614061:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234065.614062:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234065.614064:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234065.614065:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:3.0:1761234065.614067:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234065.614068:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234065.614069:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000d3898169. 00000100:00000001:3.0:1761234065.614071:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234065.614072:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.614073:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234065.614076:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fcae4007 x1846784245227648/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234090 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234065.614081:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234065.614082:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234065.614083:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234065.614085:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234065.614086:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234065.614087:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234065.614089:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234065.614090:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:3.0:1761234065.614091:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234065.614092:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234065.614093:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fcae4007. 00000100:00000001:3.0:1761234065.614095:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234065.614096:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.614098:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234065.614101:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234065.614102:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234065.614104:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234065.614105:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234066.124971:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234066.124977:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234066.124979:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234066.124983:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234066.124988:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234066.124991:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234066.124994:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234066.124997:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234066.124999:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234066.509036:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234066.509044:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234066.509055:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234066.509058:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234066.509063:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234066.509066:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234066.509129:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234066.509133:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234066.509138:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234067.149030:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234067.149036:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234067.149037:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234067.149042:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234067.149047:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234067.149051:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234067.149053:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234067.149055:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234067.149057:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234067.533017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234067.533024:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234067.533031:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234067.533033:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234067.533036:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234067.533038:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234067.533144:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234067.533150:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234067.533154:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234068.172947:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234068.172951:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234068.172953:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234068.172957:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234068.172961:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234068.172964:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234068.172966:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234068.172969:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234068.172971:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234068.556983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234068.556988:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234068.556997:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234068.556999:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234068.557004:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234068.557007:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234068.557051:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234068.557055:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234068.557057:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234069.197002:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234069.197010:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234069.197012:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234069.197018:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234069.197025:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234069.197030:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234069.197033:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234069.197036:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234069.197040:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234069.580968:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234069.580973:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234069.580980:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234069.580981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234069.580984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234069.580985:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234069.581051:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234069.581054:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234069.581057:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234070.220934:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234070.220938:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234070.220940:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234070.220943:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234070.220947:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234070.220950:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234070.220952:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234070.220953:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234070.220955:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.604960:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234070.604965:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234070.604967:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234070.604969:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.604973:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234070.604974:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.604976:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.604978:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.604981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234070.604983:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234070.604984:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234070.604989:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234070.604991:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234070.604992:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.604993:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234070.604994:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.604995:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.604996:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.604999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234070.605000:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234070.605000:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234070.605002:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234070.605003:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234070.605003:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.605004:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234070.605005:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.605006:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234070.605007:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234070.605046:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234070.605049:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234070.605052:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234070.732999:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234070.733008:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234070.733009:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234070.733012:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234070.733015:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234070.733019:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234070.733025:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6539 00000100:00080000:1.0:1761234070.733027:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234070.733030:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733032:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234070.733034:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733036:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234070.733038:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234070.733040:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234070.733042:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234070.733045:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234070.733047:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234070.733049:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234070.733052:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733053:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234070.733055:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733056:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234070.733058:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234070.733060:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733062:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234070.733064:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234070.733067:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234070.733069:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234070.733070:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234070.733072:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234070.733080:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f8a85602. 00000020:00000040:1.0:1761234070.733083:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234070.733086:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234070.733087:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234070.733089:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234070.733091:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234070.733094:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234070.733101:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761234070.733106:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234070.733112:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234070.733118:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234070.733120:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234070.733122:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f8a85602] to pc [ptlrpcd_rcv+-1] req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234070.733137:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234070.733139:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234070.733140:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234070.733143:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234070.733144:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234070.733145:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234070.733147:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234070.733149:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234070.733151:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6539 00000100:00080000:1.0:1761234070.733153:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234070.733155:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733157:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234070.733159:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733160:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234070.733162:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234070.733163:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234070.733165:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234070.733166:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234070.733168:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234070.733170:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234070.733172:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733174:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234070.733176:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733177:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234070.733179:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234070.733181:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733183:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234070.733185:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234070.733190:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234070.733192:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733192:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234070.733194:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234070.733196:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1761234070.733196:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234070.733198:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000008:00000001:1.0:1761234070.733199:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234070.733201:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234070.733202:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000010:1.0:1761234070.733206:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761234070.733208:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234070.733210:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:3.0:1761234070.733210:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1761234070.733211:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733212:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234070.733213:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234070.733214:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234070.733215:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234070.733216:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:1.0:1761234070.733218:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 02000000:00000001:3.0:1761234070.733218:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234070.733220:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8a85602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227712:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761234070.733221:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234070.733224:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733224:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234070.733228:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1761234070.733230:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1761234070.733230:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234070.733231:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234070.733233:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234070.733233:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234070.733238:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 00000100:00000001:1.0:1761234070.733240:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234070.733240:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234070.733242:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234070.733243:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1761234070.733245:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00000001:1.0:1761234070.733246:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234070.733247:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234070.733249:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234070.733249:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234070.733251:0:468:0:(import.c:519:import_select_connection()) Process entered 00000400:00000010:3.0:1761234070.733251:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00080000:1.0:1761234070.733253:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6539 00000100:00080000:1.0:1761234070.733255:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234070.733257:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000200:3.0:1761234070.733257:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227712, portal 25 00000100:00000001:1.0:1761234070.733259:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1761234070.733260:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1761234070.733261:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234070.733261:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039819776 : -104717669731840 : ffffa0c2847e7200) 00000100:00000001:1.0:1761234070.733263:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234070.733264:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234070.733265:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000100:00000040:3.0:1761234070.733265:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234095 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000020:00000001:1.0:1761234070.733267:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234070.733269:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234070.733270:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234070.733272:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:3.0:1761234070.733272:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761234070.733275:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733276:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234070.733276:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000040:1.0:1761234070.733278:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000400:00000010:3.0:1761234070.733278:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000001:1.0:1761234070.733279:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000200:3.0:1761234070.733280:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245227712, offset 0 00000020:00000040:1.0:1761234070.733281:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234070.733283:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733285:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234070.733286:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1761234070.733287:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:1.0:1761234070.733288:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234070.733289:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234070.733291:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234070.733292:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234070.733294:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234070.733295:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234070.733301:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000010:1.0:1761234070.733304:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761234070.733307:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000200:3.0:1761234070.733307:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 02000000:00000001:1.0:1761234070.733309:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234070.733310:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234070.733311:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1761234070.733312:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234070.733313:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234070.733314:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234070.733314:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 02000000:00000010:1.0:1761234070.733318:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000400:00000200:3.0:1761234070.733318:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11775:11775:256:4294967295] 192.168.201.124@tcp LPNI seq info [11775:11775:8:4294967295] 00000100:00000001:1.0:1761234070.733320:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234070.733323:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234070.733330:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1761234070.733330:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:1.0:1761234070.733331:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234070.733333:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.0:1761234070.733336:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234070.733340:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1761234070.733340:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000100:00000001:1.0:1761234070.733342:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234070.733345:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6549) 00000800:00000200:3.0:1761234070.733346:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234070.733349:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761234070.733355:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234070.733358:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1761234070.733363:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234070.733365:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234070.733370:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234070.733375:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234070.733379:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234070.733381:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234070.733385:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234070.733388:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234095 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733397:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234070.733399:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234095 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733404:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733405:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234070.733411:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733412:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234070.733414:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733416:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234070.733418:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234070/real 1761234070] req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234070.733424:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234070.733425:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234070.733426:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234070.733427:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234070.733428:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234070.733430:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234070.733431:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234070.733436:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733440:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234070.733441:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733442:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234070.733443:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234070.733445:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733449:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234070.733451:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234070.733455:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234070.733456:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733458:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234070.733458:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733460:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234070.733461:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234070.733463:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234070.733465:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234070.733466:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234070.733468:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234070.733472:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8a85602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227712:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234070.733475:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733476:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234070.733477:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f8a85602 x1846784245227712/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234070.733481:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234070.733483:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234070.733484:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234070.733486:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234070.733487:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234070.733488:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234070.733490:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234070.733491:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a174ebf2. 02000000:00000001:3.0:1761234070.733496:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234070.733497:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234070.733498:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f8a85602. 00000100:00000001:3.0:1761234070.733500:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234070.733500:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733502:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733507:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234070.733509:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733510:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234070.733511:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234070.733511:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234070.733513:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733517:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234070.733518:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234070.733519:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234070.733520:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234070.733522:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcae4007 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227776:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234070.733523:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234070.733525:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234070.733525:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234070.733526:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234070.733528:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234070.733529:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234070.733531:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234070.733533:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234070.733535:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227776, portal 10 00000100:00000001:3.0:1761234070.733536:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234070.733537:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039802752 : -104717669748864 : ffffa0c2847e2f80) 00000100:00000040:3.0:1761234070.733540:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234095 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733544:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234070.733546:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234070.733547:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234070.733548:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245227776, offset 0 00000400:00000200:3.0:1761234070.733551:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234070.733554:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234070.733557:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234070.733559:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234070.733561:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234070.733563:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11776:11776:256:4294967295] 192.168.201.124@tcp LPNI seq info [11776:11776:8:4294967295] 00000400:00000200:3.0:1761234070.733569:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234070.733572:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234070.733574:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234070.733577:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234070.733578:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761234070.733581:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234070.733582:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1761234070.733584:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234070.733586:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234070.733587:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234070.733589:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234070.733591:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234070.733592:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234070.733593:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234070.733595:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234095 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733599:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234070.733601:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234095 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733604:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733605:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234070.733607:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733608:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234070.733609:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733610:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234070.733611:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234070/real 1761234070] req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234070.733616:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234070.733617:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234070.733618:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234070.733619:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234070.733619:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234070.733620:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234070.733621:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234070.733625:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733632:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234070.733633:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733634:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234070.733635:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234070.733636:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733640:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234070.733641:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234070.733645:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234070.733646:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733647:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234070.733647:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733648:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234070.733649:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234070.733650:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234070.733651:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234070.733652:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234070.733654:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234070.733658:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcae4007 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227776:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234070.733659:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234070.733661:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733664:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234070.733665:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234070.733666:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234070.733667:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234070.733669:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d3898169 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227840:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234070.733671:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234070.733672:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234070.733701:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234070.733702:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234070.733704:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234070.733705:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234070.733707:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234070.733709:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234070.733711:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227840, portal 4 00000100:00000001:3.0:1761234070.733712:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234070.733713:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039796672 : -104717669754944 : ffffa0c2847e17c0) 00000100:00000040:3.0:1761234070.733716:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234095 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733720:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234070.733722:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234070.733723:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234070.733724:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245227840, offset 0 00000400:00000200:3.0:1761234070.733726:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234070.733729:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234070.733731:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234070.733732:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234070.733734:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234070.733735:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11777:11777:256:4294967295] 192.168.201.124@tcp LPNI seq info [11777:11777:8:4294967295] 00000400:00000200:3.0:1761234070.733740:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234070.733743:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234070.733745:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234070.733747:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234070.733748:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761234070.733751:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234070.733752:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1761234070.733755:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234070.733756:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234070.733758:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234070.733760:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234070.733762:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234070.733763:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234070.733764:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234070.733765:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234095 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733769:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234070.733771:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234095 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733774:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733775:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234070.733777:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733778:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234070.733779:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733780:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234070.733781:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234070/real 1761234070] req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234070.733786:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234070.733787:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234070.733787:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234070.733788:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234070.733789:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234070.733790:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234070.733791:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234070.733795:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733798:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234070.733799:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733800:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234070.733801:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234070.733802:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234070.733810:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234070.733811:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234070.733815:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234070.733816:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733817:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234070.733818:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733819:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234070.733819:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234070.733820:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234070.733822:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234070.733823:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234070.733824:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234070.733828:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d3898169 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227840:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234070.733830:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733831:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234070.733832:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fcae4007 x1846784245227776/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234070.733835:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234070.733836:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234070.733837:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234070.733838:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234070.733839:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234070.733840:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234070.733841:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234070.733842:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:3.0:1761234070.733863:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234070.733864:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234070.733865:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fcae4007. 00000100:00000001:3.0:1761234070.733867:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234070.733868:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733869:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234070.733871:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000d3898169 x1846784245227840/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234095 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234070.733875:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234070.733876:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234070.733876:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234070.733879:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234070.733879:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234070.733880:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234070.733881:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234070.733881:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:3.0:1761234070.733882:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234070.733883:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234070.733884:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000d3898169. 00000100:00000001:3.0:1761234070.733885:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234070.733885:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733886:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234070.733889:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234070.733890:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234070.733891:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234070.733892:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234071.244992:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234071.244998:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234071.245001:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234071.245005:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234071.245010:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234071.245014:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234071.245016:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234071.245018:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234071.245020:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234071.629004:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234071.629011:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234071.629020:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234071.629023:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234071.629028:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234071.629031:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234071.629187:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234071.629193:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234071.629199:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234072.268934:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234072.268941:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234072.268943:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234072.268949:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234072.268954:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234072.268957:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234072.268960:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234072.268962:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234072.268964:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234072.652956:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234072.652960:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234072.652965:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234072.652966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234072.652969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234072.652970:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234072.653041:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234072.653044:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234072.653047:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234073.293009:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234073.293016:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234073.293018:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234073.293023:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234073.293028:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234073.293032:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234073.293035:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234073.293037:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234073.293039:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234073.677010:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234073.677021:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234073.677034:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234073.677037:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234073.677043:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234073.677045:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234073.677196:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234073.677202:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234073.677208:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234074.316993:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234074.317001:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234074.317004:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234074.317012:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234074.317018:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234074.317023:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234074.317027:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234074.317030:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234074.317035:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234074.701073:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234074.701080:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234074.701090:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234074.701093:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234074.701099:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234074.701100:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234074.701238:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234074.701243:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234074.701247:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234075.340994:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234075.340999:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234075.341000:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234075.341005:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234075.341009:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234075.341012:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234075.341015:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234075.341017:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234075.341019:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234075.724981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234075.724988:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234075.724995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234075.724997:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234075.725000:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234075.725001:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234075.725076:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234075.725081:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234075.725087:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234075.853004:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234075.853017:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234075.853019:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234075.853022:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234075.853025:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234075.853031:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234075.853038:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6544 00000100:00080000:1.0:1761234075.853042:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234075.853045:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853049:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234075.853053:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853056:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234075.853060:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234075.853062:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234075.853066:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234075.853069:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234075.853073:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234075.853076:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234075.853080:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853082:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234075.853085:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853087:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234075.853090:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234075.853094:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853096:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234075.853099:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234075.853103:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234075.853106:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853108:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234075.853121:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234075.853131:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761234075.853135:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234075.853139:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234075.853141:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853144:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234075.853147:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234075.853152:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234075.853161:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761234075.853168:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234075.853176:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234075.853188:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234075.853192:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234075.853196:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234075.853222:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853224:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234075.853226:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234075.853230:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234075.853232:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234075.853234:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234075.853235:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234075.853239:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234075.853241:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6544 00000100:00080000:1.0:1761234075.853244:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234075.853248:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853250:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234075.853253:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853255:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234075.853258:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234075.853259:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234075.853261:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234075.853263:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234075.853265:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234075.853268:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234075.853271:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853273:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234075.853275:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853276:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234075.853279:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234075.853282:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853285:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234075.853287:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234075.853294:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234075.853297:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853299:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234075.853303:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234075.853307:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853309:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.853312:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1761234075.853314:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000100:00000001:3.0:1761234075.853317:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1761234075.853319:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:3.0:1761234075.853321:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234075.853326:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234075.853328:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234075.853329:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234075.853331:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234075.853334:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234075.853336:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234075.853343:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:3.0:1761234075.853344:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234075.853346:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853351:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234075.853352:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234075.853371:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234075.853376:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761234075.853378:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003630282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227904:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234075.853389:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234075.853393:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1761234075.853395:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1761234075.853396:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853398:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234075.853400:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1761234075.853408:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:3.0:1761234075.853408:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234075.853413:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234075.853421:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00000001:1.0:1761234075.853428:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234075.853430:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000001:1.0:1761234075.853433:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234075.853437:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:3.0:1761234075.853439:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227904, portal 25 00000100:00000001:1.0:1761234075.853444:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00000001:3.0:1761234075.853445:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1761234075.853448:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234075.853449:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039816128 : -104717669735488 : ffffa0c2847e63c0) 00000100:00000001:1.0:1761234075.853453:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234075.853456:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1761234075.853458:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234100 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234075.853463:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:3.0:1761234075.853469:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00080000:1.0:1761234075.853470:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6544 00000100:00080000:1.0:1761234075.853477:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234075.853477:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234075.853482:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000040:1.0:1761234075.853486:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000200:3.0:1761234075.853488:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245227904, offset 0 00000100:00000001:1.0:1761234075.853492:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234075.853498:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000400:00000200:3.0:1761234075.853500:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853504:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234075.853508:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234075.853509:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234075.853511:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234075.853514:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234075.853517:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000200:3.0:1761234075.853519:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000020:00000001:1.0:1761234075.853521:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234075.853529:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000200:3.0:1761234075.853532:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000001:1.0:1761234075.853535:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1761234075.853538:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1761234075.853558:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000400:00000200:3.0:1761234075.853562:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000001:1.0:1761234075.853564:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234075.853569:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000200:3.0:1761234075.853570:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11778:11778:256:4294967295] 192.168.201.124@tcp LPNI seq info [11778:11778:8:4294967295] 00000100:00080000:1.0:1761234075.853601:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000200:3.0:1761234075.853605:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:1.0:1761234075.853608:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234075.853613:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.0:1761234075.853614:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 02000000:00000001:1.0:1761234075.853619:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:3.0:1761234075.853620:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 02000000:00000001:1.0:1761234075.853627:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853631:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000800:00000200:3.0:1761234075.853632:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000008:00100000:1.0:1761234075.853634:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000800:00000200:3.0:1761234075.853637:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000008:00000001:1.0:1761234075.853639:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853644:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761234075.853645:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234075.853649:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000100:00000010:1.0:1761234075.853651:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000800:00000100:3.0:1761234075.853655:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000020:00000040:1.0:1761234075.853658:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000800:00000010:3.0:1761234075.853659:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 02000000:00000001:1.0:1761234075.853665:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000400:00000200:3.0:1761234075.853668:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1761234075.853669:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853674:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234075.853677:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1761234075.853677:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1761234075.853681:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234075.853683:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234075.853687:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 02000000:00000010:1.0:1761234075.853689:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:3.0:1761234075.853691:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234075.853697:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234100 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234075.853698:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234075.853709:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.853710:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234075.853717:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234100 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234075.853738:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234075.853739:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853742:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.853743:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234075.853750:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234075.853752:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.853754:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234075.853759:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.853764:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234075.853770:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234075/real 1761234075] req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234075.853774:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234075.853779:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000001:3.0:1761234075.853783:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1761234075.853786:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6554) 00000400:00000001:3.0:1761234075.853786:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234075.853789:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234075.853792:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234075.853793:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234075.853795:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234075.853798:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234075.853804:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.853810:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234075.853812:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.853814:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234075.853816:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234075.853818:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.853824:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234075.853827:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234075.853833:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234075.853835:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.853836:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234075.853837:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.853839:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234075.853840:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234075.853868:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234075.853871:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234075.853873:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234075.853877:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234075.853883:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003630282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245227904:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234075.853887:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.853889:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234075.853890:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000003630282c x1846784245227904/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234075.853895:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234075.853897:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234075.853899:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234075.853903:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234075.853905:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234075.853906:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234075.853908:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234075.853910:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006d8d01a9. 02000000:00000001:3.0:1761234075.853913:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234075.853915:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234075.853916:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000003630282c. 00000100:00000001:3.0:1761234075.853919:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234075.853920:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.853921:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.853927:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234075.853929:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.853930:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234075.853932:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234075.853933:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234075.853935:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.853940:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234075.853941:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234075.853943:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234075.853944:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234075.853946:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f4f489ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227968:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234075.853949:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234075.853950:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234075.853951:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234075.853953:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234075.853955:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234075.853956:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234075.853959:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234075.853961:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234075.853963:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245227968, portal 10 00000100:00000001:3.0:1761234075.853965:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234075.853966:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039793024 : -104717669758592 : ffffa0c2847e0980) 00000100:00000040:3.0:1761234075.853970:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234100 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.853975:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234075.853977:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234075.853979:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234075.853981:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245227968, offset 0 00000400:00000200:3.0:1761234075.853984:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234075.853989:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234075.853992:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234075.853994:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234075.853996:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234075.853998:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11779:11779:256:4294967295] 192.168.201.124@tcp LPNI seq info [11779:11779:8:4294967295] 00000400:00000200:3.0:1761234075.854005:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234075.854008:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234075.854010:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234075.854013:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234075.854015:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761234075.854018:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234075.854020:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1761234075.854023:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234075.854024:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234075.854026:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234075.854029:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234075.854031:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234075.854032:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234075.854034:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234075.854036:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234100 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854041:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234075.854043:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234100 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854048:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.854049:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234075.854051:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.854052:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234075.854054:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.854055:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234075.854057:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234075/real 1761234075] req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234075.854063:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234075.854064:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234075.854065:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234075.854066:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234075.854067:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234075.854068:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234075.854070:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234075.854075:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854079:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234075.854080:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854081:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234075.854082:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234075.854084:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854089:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234075.854091:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234075.854096:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234075.854097:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854098:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234075.854099:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854100:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234075.854102:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234075.854103:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234075.854105:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234075.854106:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234075.854108:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234075.854119:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f4f489ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245227968:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234075.854122:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234075.854123:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854128:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234075.854129:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234075.854130:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234075.854131:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234075.854133:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3b47c4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228032:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234075.854135:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234075.854136:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234075.854137:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234075.854138:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234075.854140:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234075.854141:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234075.854143:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234075.854145:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234075.854147:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228032, portal 4 00000100:00000001:3.0:1761234075.854149:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234075.854150:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039818560 : -104717669733056 : ffffa0c2847e6d40) 00000100:00000040:3.0:1761234075.854153:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234100 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854157:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234075.854159:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234075.854161:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234075.854162:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245228032, offset 0 00000400:00000200:3.0:1761234075.854165:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234075.854168:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234075.854171:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234075.854172:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234075.854174:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234075.854176:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11780:11780:256:4294967295] 192.168.201.124@tcp LPNI seq info [11780:11780:8:4294967295] 00000400:00000200:3.0:1761234075.854183:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234075.854186:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234075.854188:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234075.854191:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234075.854192:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761234075.854195:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234075.854197:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1761234075.854199:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234075.854200:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234075.854202:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234075.854205:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234075.854207:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234075.854208:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234075.854209:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234075.854211:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234100 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854216:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234075.854218:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234100 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854223:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.854224:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234075.854226:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.854227:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234075.854228:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.854229:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234075.854231:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234075/real 1761234075] req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234075.854236:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234075.854237:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234075.854237:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234075.854239:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234075.854240:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234075.854241:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234075.854243:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234075.854248:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854253:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234075.854253:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854255:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234075.854255:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234075.854257:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234075.854262:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234075.854264:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234075.854268:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234075.854269:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854270:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234075.854271:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854272:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234075.854273:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234075.854275:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234075.854276:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234075.854277:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234075.854279:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234075.854284:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3b47c4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228032:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234075.854287:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854288:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234075.854290:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f4f489ba x1846784245227968/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234075.854295:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234075.854296:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234075.854297:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234075.854299:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234075.854300:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234075.854301:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234075.854302:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234075.854303:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:3.0:1761234075.854305:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234075.854306:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234075.854307:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f4f489ba. 00000100:00000001:3.0:1761234075.854308:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234075.854309:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854311:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234075.854312:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000c3b47c4a x1846784245228032/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234100 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234075.854316:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234075.854317:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234075.854318:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234075.854319:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234075.854320:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234075.854320:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234075.854322:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234075.854322:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:3.0:1761234075.854324:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234075.854324:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234075.854325:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000c3b47c4a. 00000100:00000001:3.0:1761234075.854327:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234075.854327:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854329:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234075.854331:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234075.854332:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234075.854333:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234075.854334:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234076.364913:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234076.364919:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234076.364920:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234076.364925:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234076.364929:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234076.364933:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234076.364935:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234076.364937:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234076.364939:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234076.749027:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234076.749032:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234076.749040:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234076.749042:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234076.749046:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234076.749048:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234076.749163:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234076.749168:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234076.749173:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234077.388997:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234077.389002:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234077.389003:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234077.389007:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234077.389010:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234077.389013:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234077.389015:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234077.389016:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234077.389018:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234077.772967:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234077.772973:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234077.772979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234077.772980:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234077.772983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234077.772985:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234077.773086:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234077.773091:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234077.773095:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234078.412962:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234078.412967:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234078.412968:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234078.412972:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234078.412976:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234078.412979:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234078.412981:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234078.412982:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234078.412984:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234078.797040:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234078.797049:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234078.797058:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234078.797060:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234078.797065:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234078.797067:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234078.797246:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234078.797250:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234078.797255:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234079.436954:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234079.436959:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234079.436960:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234079.436964:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234079.436969:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234079.436972:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234079.436974:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234079.436975:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234079.436977:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234079.820929:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234079.820933:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234079.820938:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234079.820939:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234079.820941:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234079.820942:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234079.821020:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234079.821024:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234079.821028:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234080.461053:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234080.461061:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234080.461063:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234080.461068:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234080.461073:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234080.461077:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234080.461080:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234080.461083:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234080.461085:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845036:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234080.845045:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234080.845048:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234080.845051:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845058:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234080.845061:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845063:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845066:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845070:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234080.845073:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234080.845074:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234080.845080:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234080.845082:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234080.845084:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845086:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234080.845087:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845089:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845091:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845094:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234080.845096:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234080.845097:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234080.845100:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234080.845101:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234080.845102:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845104:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234080.845105:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845107:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234080.845110:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234080.845223:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234080.845228:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234080.845234:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234080.972978:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234080.972992:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234080.972994:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234080.972997:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234080.973000:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234080.973007:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234080.973014:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6549 00000100:00080000:0.0:1761234080.973017:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234080.973020:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973023:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234080.973026:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973027:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234080.973031:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234080.973033:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234080.973035:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234080.973039:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234080.973040:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234080.973042:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234080.973044:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973046:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234080.973047:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973048:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234080.973050:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234080.973053:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973055:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234080.973057:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234080.973060:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234080.973062:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973064:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234080.973067:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234080.973077:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761234080.973081:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234080.973086:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234080.973087:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973090:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234080.973094:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234080.973098:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234080.973107:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761234080.973113:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234080.973122:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234080.973134:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234080.973136:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234080.973141:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234080.973163:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973165:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234080.973168:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234080.973172:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234080.973173:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234080.973174:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234080.973175:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234080.973177:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234080.973179:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6549 00000100:00080000:0.0:1761234080.973181:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234080.973182:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973184:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234080.973185:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973186:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234080.973187:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234080.973188:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234080.973190:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234080.973192:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234080.973194:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234080.973195:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234080.973197:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973198:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234080.973200:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973201:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234080.973202:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234080.973204:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973205:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234080.973207:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234080.973212:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234080.973214:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973215:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234080.973217:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1761234080.973219:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234080.973222:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973223:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234080.973224:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973227:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000010:0.0:1761234080.973228:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761234080.973231:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234080.973234:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000040:3.0:1761234080.973234:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:0.0:1761234080.973235:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973238:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234080.973239:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234080.973241:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973246:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 02000000:00000010:0.0:1761234080.973247:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:3.0:1761234080.973248:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973253:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234080.973255:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234080.973258:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234080.973260:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761234080.973264:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e728cb50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228096:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234080.973272:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1761234080.973274:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234080.973275:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234080.973277:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1761234080.973279:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234080.973280:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234080.973285:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1761234080.973289:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973290:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 02000000:00000010:3.0:1761234080.973291:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 00000100:00080000:0.0:1761234080.973292:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1761234080.973295:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973296:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234080.973298:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761234080.973299:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234080.973300:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234080.973302:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000400:00000010:3.0:1761234080.973303:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000100:00080000:0.0:1761234080.973304:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6549 00000100:00080000:0.0:1761234080.973308:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234080.973310:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973311:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234080.973312:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973313:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000010:3.0:1761234080.973313:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000020:00000001:0.0:1761234080.973317:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234080.973318:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234080.973319:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234080.973320:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761234080.973321:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761234080.973323:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000200:3.0:1761234080.973323:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228096, portal 25 00000100:00000040:0.0:1761234080.973328:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234080.973328:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1761234080.973330:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1761234080.973331:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246295744 : -104717463255872 : ffffa0c290cd04c0) 00000100:00000040:0.0:1761234080.973333:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973336:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234080.973338:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00000040:3.0:1761234080.973341:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234105 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761234080.973342:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234080.973347:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234080.973350:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234080.973352:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234080.973354:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 02000000:00000001:0.0:1761234080.973355:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973357:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234080.973359:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000040:3.0:1761234080.973359:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000008:00000001:0.0:1761234080.973363:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234080.973363:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000001:0.0:1761234080.973365:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1761234080.973367:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245228096, offset 0 00000100:00000010:0.0:1761234080.973370:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761234080.973372:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234080.973374:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234080.973375:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973377:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234080.973378:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1761234080.973379:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:0.0:1761234080.973380:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234080.973385:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761234080.973388:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234080.973391:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234080.973396:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234080.973397:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234080.973397:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000040:0.0:1761234080.973402:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234080.973407:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234080.973409:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000200:3.0:1761234080.973410:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000040:0.0:1761234080.973412:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6560) 00000400:00000200:3.0:1761234080.973416:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234080.973419:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234080.973423:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11781:11781:256:4294967295] 192.168.201.124@tcp LPNI seq info [11781:11781:8:4294967295] 00000400:00000200:3.0:1761234080.973433:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234080.973437:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234080.973442:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234080.973449:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234080.973451:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761234080.973457:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234080.973459:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1761234080.973462:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234080.973465:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234080.973471:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234080.973475:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234080.973493:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234080.973497:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234080.973502:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234080.973505:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234105 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973513:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234080.973514:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234105 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973519:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973521:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234080.973527:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973528:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234080.973530:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973533:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234080.973535:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234080/real 1761234080] req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234080.973543:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234080.973545:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234080.973547:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234080.973548:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234080.973549:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234080.973551:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234080.973552:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234080.973557:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973561:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234080.973562:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973563:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234080.973565:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234080.973567:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973571:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234080.973574:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234080.973578:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234080.973581:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973582:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234080.973582:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973584:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234080.973586:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234080.973588:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234080.973589:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234080.973591:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234080.973593:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234080.973598:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e728cb50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228096:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234080.973601:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973603:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234080.973604:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000e728cb50 x1846784245228096/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234080.973609:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234080.973612:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234080.973614:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234080.973616:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234080.973617:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234080.973618:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234080.973620:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234080.973621:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000040aa0f54. 02000000:00000001:3.0:1761234080.973625:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234080.973626:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234080.973628:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000e728cb50. 00000100:00000001:3.0:1761234080.973630:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234080.973631:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973632:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973641:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234080.973642:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973644:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234080.973645:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234080.973646:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234080.973647:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973652:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234080.973653:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234080.973654:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234080.973656:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234080.973657:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc434e0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228160:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234080.973659:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234080.973660:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234080.973661:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234080.973662:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234080.973664:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234080.973665:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234080.973668:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234080.973670:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234080.973672:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228160, portal 10 00000100:00000001:3.0:1761234080.973673:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234080.973674:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246323712 : -104717463227904 : ffffa0c290cd7200) 00000100:00000040:3.0:1761234080.973679:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234105 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973683:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234080.973685:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234080.973686:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234080.973688:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245228160, offset 0 00000400:00000200:3.0:1761234080.973691:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234080.973694:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234080.973696:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234080.973698:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234080.973700:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234080.973702:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11782:11782:256:4294967295] 192.168.201.124@tcp LPNI seq info [11782:11782:8:4294967295] 00000400:00000200:3.0:1761234080.973707:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234080.973710:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234080.973712:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234080.973715:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234080.973717:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761234080.973719:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234080.973721:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1761234080.973723:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234080.973724:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234080.973726:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234080.973728:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234080.973731:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234080.973732:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234080.973733:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234080.973734:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234105 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973739:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234080.973741:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234105 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973745:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973746:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234080.973748:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973749:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234080.973750:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973751:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234080.973752:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234080/real 1761234080] req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234080.973757:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234080.973758:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234080.973759:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234080.973760:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234080.973761:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234080.973762:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234080.973763:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234080.973767:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973770:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234080.973771:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973772:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234080.973772:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234080.973774:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973778:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234080.973779:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234080.973784:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234080.973785:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973786:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234080.973788:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973789:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234080.973790:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234080.973792:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234080.973795:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234080.973797:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234080.973800:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234080.973808:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc434e0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228160:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234080.973812:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234080.973814:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973821:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234080.973822:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234080.973824:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234080.973826:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234080.973828:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000039bf264 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228224:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234080.973832:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234080.973833:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234080.973834:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234080.973835:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234080.973838:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234080.973839:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234080.973867:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234080.973871:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234080.973873:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228224, portal 4 00000100:00000001:3.0:1761234080.973874:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234080.973875:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246311552 : -104717463240064 : ffffa0c290cd4280) 00000100:00000040:3.0:1761234080.973879:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234105 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973884:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234080.973886:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234080.973887:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000200:3.0:1761234080.973888:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245228224, offset 0 00000400:00000200:3.0:1761234080.973891:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234080.973895:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234080.973897:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234080.973899:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234080.973901:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234080.973903:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11783:11783:256:4294967295] 192.168.201.124@tcp LPNI seq info [11783:11783:8:4294967295] 00000400:00000200:3.0:1761234080.973908:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234080.973911:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234080.973912:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234080.973915:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234080.973917:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761234080.973920:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234080.973921:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1761234080.973924:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234080.973925:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676282). 00000400:00000200:3.0:1761234080.973927:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234080.973930:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234080.973931:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:3.0:1761234080.973932:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:3.0:1761234080.973934:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234080.973935:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234105 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973940:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234080.973942:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234105 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973946:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973947:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234080.973948:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973949:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234080.973950:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.973952:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234080.973954:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234080/real 1761234080] req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234080.973959:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234080.973960:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234080.973961:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234080.973962:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234080.973963:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234080.973964:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234080.973965:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234080.973970:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973973:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234080.973974:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973974:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234080.973975:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234080.973977:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234080.973980:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234080.973982:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234080.973985:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234080.973987:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973988:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234080.973989:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.973990:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234080.973991:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234080.973992:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234080.973994:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234080.973995:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234080.973997:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234080.974002:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000039bf264 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228224:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234080.974004:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.974005:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234080.974006:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000dc434e0c x1846784245228160/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234080.974010:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234080.974011:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234080.974011:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234080.974013:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234080.974013:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234080.974014:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234080.974015:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234080.974016:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078138607. 02000000:00000001:3.0:1761234080.974017:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234080.974018:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234080.974019:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000dc434e0c. 00000100:00000001:3.0:1761234080.974021:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234080.974021:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.974023:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234080.974024:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000039bf264 x1846784245228224/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234105 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234080.974028:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234080.974028:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234080.974029:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234080.974031:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234080.974032:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234080.974032:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234080.974033:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234080.974034:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:3.0:1761234080.974035:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234080.974036:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234080.974036:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000039bf264. 00000100:00000001:3.0:1761234080.974038:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234080.974039:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.974040:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234080.974042:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234080.974043:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234080.974045:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234080.974046:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234081.485017:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234081.485023:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234081.485025:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234081.485030:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234081.485036:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234081.485040:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234081.485043:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234081.485045:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234081.485048:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234081.869031:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234081.869037:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234081.869047:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234081.869048:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234081.869051:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234081.869052:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234081.869163:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234081.869167:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234081.869172:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234082.508989:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234082.508996:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234082.508999:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234082.509004:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234082.509009:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234082.509013:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234082.509016:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234082.509018:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234082.509021:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234082.892950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234082.892954:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234082.892959:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234082.892960:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234082.892963:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234082.892964:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234082.893047:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234082.893051:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234082.893054:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234083.533002:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234083.533008:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234083.533009:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234083.533014:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234083.533019:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234083.533023:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234083.533026:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234083.533029:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234083.533031:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234083.917008:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234083.917014:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234083.917023:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234083.917024:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234083.917028:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234083.917029:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234083.917070:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234083.917076:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234083.917081:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234084.557069:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234084.557074:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234084.557076:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234084.557080:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234084.557086:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234084.557089:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234084.557103:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234084.557121:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234084.557123:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234084.940990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234084.940997:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234084.941006:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234084.941008:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234084.941012:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234084.941013:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234084.941147:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234084.941173:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234084.941180:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234085.580949:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234085.580954:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234085.580955:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234085.580959:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234085.580964:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234085.580967:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234085.580970:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234085.580972:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234085.580975:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234085.964984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234085.964990:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234085.964997:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234085.964998:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234085.965001:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234085.965002:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234085.965032:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234085.965036:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234085.965039:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761234086.093035:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234086.093044:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234086.093045:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:2.0:1761234086.093047:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234086.093049:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234086.093054:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234086.093059:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6555 00000100:00080000:2.0:1761234086.093062:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234086.093064:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093066:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234086.093067:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093069:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234086.093071:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234086.093073:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:2.0:1761234086.093075:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234086.093077:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:2.0:1761234086.093079:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:2.0:1761234086.093080:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:2.0:1761234086.093082:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093083:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234086.093084:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093086:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234086.093088:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:2.0:1761234086.093090:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093092:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234086.093094:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234086.093096:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234086.093098:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093099:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234086.093101:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234086.093108:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000598041cd. 00000020:00000040:2.0:1761234086.093111:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:2.0:1761234086.093114:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234086.093115:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093118:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234086.093120:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234086.093123:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234086.093129:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000062f3eded. 00000100:00000001:2.0:1761234086.093134:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234086.093140:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234086.093146:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234086.093148:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234086.093151:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000598041cd] to pc [ptlrpcd_rcv+-1] req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234086.093168:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093169:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234086.093170:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234086.093173:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234086.093173:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1761234086.093174:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234086.093175:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234086.093176:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234086.093178:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6555 00000100:00080000:2.0:1761234086.093180:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234086.093182:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093183:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234086.093184:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093185:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234086.093186:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234086.093187:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:2.0:1761234086.093187:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234086.093189:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:2.0:1761234086.093190:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:2.0:1761234086.093191:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:2.0:1761234086.093192:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093193:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234086.093194:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093195:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234086.093196:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:2.0:1761234086.093197:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093199:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234086.093200:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234086.093204:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234086.093205:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093206:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234086.093208:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234086.093210:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093211:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234086.093213:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e8dfec2c. 00000020:00000040:2.0:1761234086.093215:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:2.0:1761234086.093216:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234086.093217:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093218:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234086.093219:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234086.093219:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234086.093221:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f7091f62. 00000100:00000001:2.0:1761234086.093224:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234086.093226:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234086.093231:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234086.093232:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234086.093234:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e8dfec2c] to pc [ptlrpcd_rcv+-1] req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234086.093239:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093240:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234086.093241:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234086.093243:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234086.093243:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1761234086.093244:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1761234086.093244:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1761234086.093245:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234086.093248:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:3.0:1761234086.093249:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1761234086.093251:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6555 00000100:00080000:2.0:1761234086.093253:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1761234086.093253:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234086.093256:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093257:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234086.093260:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000040:3.0:1761234086.093260:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234086.093263:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234086.093267:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234086.093268:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:2.0:1761234086.093269:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234086.093271:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:2.0:1761234086.093273:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:3.0:1761234086.093273:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234086.093275:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234086.093276:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234086.093279:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000001:3.0:1761234086.093280:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1761234086.093281:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1761234086.093283:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234086.093284:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093288:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234086.093290:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00100000:3.0:1761234086.093290:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000598041cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228288:192.168.201.124@tcp:250: 00000100:00080000:2.0:1761234086.093296:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234086.093296:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1761234086.093299:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234086.093300:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234086.093301:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234086.093303:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234086.093304:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234086.093306:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1761234086.093307:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093309:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234086.093310:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 02000000:00000010:3.0:1761234086.093311:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 00000008:00000001:2.0:1761234086.093312:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234086.093315:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093316:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234086.093321:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000032f5ca64. 00000020:00000040:2.0:1761234086.093322:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000010:3.0:1761234086.093323:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 02000000:00000001:2.0:1761234086.093324:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234086.093325:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.093327:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234086.093328:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234086.093329:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234086.093331:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 02000000:00000010:2.0:1761234086.093332:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006ca8bb96. 00000100:00000200:3.0:1761234086.093337:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228288, portal 25 00000100:00000001:3.0:1761234086.093340:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234086.093341:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090831040 : -104717618720576 : ffffa0c28788d0c0) 00000100:00000040:3.0:1761234086.093345:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234111 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234086.093346:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234086.093353:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234086.093357:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:2.0:1761234086.093360:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1761234086.093361:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093362:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234086.093365:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000bb195ee3. 00000100:00000040:2.0:1761234086.093366:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000032f5ca64] to pc [ptlrpcd_rcv+-1] req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234086.093369:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245228288, offset 0 00000100:00000001:2.0:1761234086.093377:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234086.093379:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:2.0:1761234086.093395:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:2.0:1761234086.093400:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6565) 00000400:00000200:3.0:1761234086.093409:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234086.093416:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234086.093420:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234086.093422:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234086.093426:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11784:11784:256:4294967295] 192.168.201.124@tcp LPNI seq info [11784:11784:8:4294967295] 00000400:00000200:3.0:1761234086.093433:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234086.093437:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234086.093440:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000008e257ae2 (tot 1676530). 00000800:00000200:3.0:1761234086.093445:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234086.093449:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761234086.093466:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234086.093467:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234086.093469:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234086.093471:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234086.093473:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234086.093477:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234086.093478:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234086.093479:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234086.093480:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234086.093482:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8dfec2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228352:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234086.093484:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234086.093485:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234086.093485:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234086.093486:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234086.093488:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234086.093489:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234086.093491:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000009aaa4f6b. 00000400:00000010:3.0:1761234086.093492:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234086.093495:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228352, portal 10 00000100:00000001:3.0:1761234086.093496:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234086.093497:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090837120 : -104717618714496 : ffffa0c28788e880) 00000100:00000040:3.0:1761234086.093500:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234111 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234086.093505:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234086.093507:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234086.093509:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000092f1a425. 00000100:00000200:3.0:1761234086.093510:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245228352, offset 0 00000400:00000200:3.0:1761234086.093514:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234086.093516:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234086.093519:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234086.093521:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234086.093522:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234086.093524:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11785:11785:255:4294967295] 192.168.201.124@tcp LPNI seq info [11785:11785:7:4294967295] 00000400:00000200:3.0:1761234086.093529:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234086.093532:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234086.093534:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000006315f304 (tot 1676778). 00000800:00000200:3.0:1761234086.093537:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234086.093540:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761234086.093542:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234086.093543:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234086.093544:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234086.093546:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234086.093547:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234086.093553:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234086.093554:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1761234086.093556:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234086.093558:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234086.093559:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234086.093560:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234086.093564:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234086.093565:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234086.093567:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234086.093568:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1761234086.093569:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000100:00100000:3.0:1761234086.093570:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032f5ca64 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228416:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234086.093575:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234086.093576:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234086.093577:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234086.093578:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234086.093581:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000df51b12a. 02000000:00000001:3.0:1761234086.093582:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234086.093584:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234086.093586:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234086.093588:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228416, portal 4 00000100:00000001:3.0:1761234086.093589:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234086.093589:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090812800 : -104717618738816 : ffffa0c287888980) 00000100:00000040:3.0:1761234086.093593:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234111 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234086.093597:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234086.093599:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234086.093601:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000df0e2612. 00000100:00000200:3.0:1761234086.093603:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245228416, offset 0 00000400:00000200:3.0:1761234086.093607:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234086.093610:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234086.093612:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234086.093613:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234086.093614:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234086.093618:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11786:11786:254:4294967295] 192.168.201.124@tcp LPNI seq info [11786:11786:6:4294967295] 00000400:00000200:1.0:1761234086.093625:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:3.0:1761234086.093626:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234086.093633:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234086.093636:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aaa22236 (tot 1677026). 00000800:00000200:3.0:1761234086.093640:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234086.093643:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761234086.093644:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234086.093646:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234086.093647:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1761234086.093649:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000100:00000001:3.0:1761234086.093650:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234086.093653:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234086.093656:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234086.093657:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1761234086.093660:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234086.093661:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234086.093663:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234086.093665:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234086.604967:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234086.604972:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234086.604974:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234086.604979:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234086.604983:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234086.604985:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234086.604987:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234086.604989:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234086.604991:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234086.988961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234086.988982:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234086.988990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234086.988992:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234086.988995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234086.988996:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234086.989097:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234086.989101:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234086.989104:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234087.628922:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234087.628929:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234087.628931:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234087.628935:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234087.628940:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234087.628943:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234087.628946:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234087.628948:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234087.628950:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234088.012985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234088.012989:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234088.012996:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234088.012997:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234088.012999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234088.013000:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234088.013083:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234088.013088:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234088.013092:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234088.653062:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234088.653070:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234088.653072:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234088.653078:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234088.653084:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234088.653088:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234088.653091:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234088.653094:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234088.653097:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234089.036950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234089.036955:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234089.036960:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234089.036961:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234089.036964:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234089.036965:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234089.037053:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234089.037056:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234089.037058:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1761234089.165144:0:88985:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:1.0:1761234089.165182:0:88985:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1761234089.165189:0:88985:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:1.0:1761234089.165198:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:1.0:1761234089.165200:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000008e257ae2 (tot 1676778). 00000400:00000200:1.0:1761234089.165206:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1761234089.165210:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1761234089.165214:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000bb195ee3 00000400:00000010:1.0:1761234089.165215:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000bb195ee3. 00000100:00000001:1.0:1761234089.165219:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761234089.165226:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234111 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234089.165254:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761234089.165258:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234111 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234089.165265:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234089.165267:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:1.0:1761234089.165278:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:1.0:1761234089.165281:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000006315f304 (tot 1676530). 00000400:00000200:1.0:1761234089.165283:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1761234089.165287:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1761234089.165290:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000092f1a425 00000400:00000010:1.0:1761234089.165292:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000092f1a425. 00000100:00000001:1.0:1761234089.165295:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761234089.165299:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234111 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234089.165307:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:1.0:1761234089.165309:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234111 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234089.165317:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234089.165317:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234089.165320:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234089.165337:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000800:00000100:1.0:1761234089.165343:0:88985:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000100:00000001:3.0:1761234089.165343:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000800:00000010:1.0:1761234089.165347:0:88985:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aaa22236 (tot 1676282). 00000400:00000200:1.0:1761234089.165351:0:88985:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:3.0:1761234089.165353:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234086/real 1761234089] req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000200:1.0:1761234089.165358:0:88985:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1761234089.165363:0:88985:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000df0e2612 00000400:00000010:1.0:1761234089.165365:0:88985:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000df0e2612. 00000100:00000001:1.0:1761234089.165368:0:88985:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:1.0:1761234089.165373:0:88985:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234111 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234089.165375:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234089.165378:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234089.165380:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000100:00000001:1.0:1761234089.165384:0:88985:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000400:00000200:3.0:1761234089.165387:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000100:00000040:1.0:1761234089.165388:0:88985:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234111 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000010:3.0:1761234089.165390:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234089.165395:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000001:1.0:1761234089.165399:0:88985:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1761234089.165402:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234089.165403:0:88985:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000200:3.0:1761234089.165413:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234089.165421:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234089.165423:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165426:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234089.165428:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234089.165432:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234089.165439:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234089.165444:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234089.165451:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234089.165454:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165456:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234089.165456:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165460:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234089.165462:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234089.165465:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234089.165468:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234089.165471:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234089.165474:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234089.165482:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000598041cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228288:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234089.165486:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234089.165489:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234086/real 1761234089] req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234089.165495:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234089.165497:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234089.165499:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000009aaa4f6b. 00000400:00000200:3.0:1761234089.165501:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234089.165502:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234089.165504:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234089.165506:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234089.165512:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234089.165518:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234089.165519:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165520:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234089.165522:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234089.165524:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234089.165530:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234089.165533:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234089.165539:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234089.165541:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165543:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234089.165543:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165545:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234089.165546:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234089.165548:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234089.165551:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234089.165553:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234089.165556:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234089.165563:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8dfec2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228352:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234089.165568:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234089.165570:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234089.165572:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000598041cd x1846784245228288/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234089.165578:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234089.165581:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234089.165585:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234089.165591:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234089.165593:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234089.165595:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234089.165598:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234089.165600:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062f3eded. 02000000:00000001:3.0:1761234089.165605:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234089.165607:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234089.165608:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000598041cd. 00000100:00000001:3.0:1761234089.165613:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234089.165614:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165615:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234089.165618:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000e8dfec2c x1846784245228352/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234089.165623:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234089.165624:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234089.165625:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234089.165628:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234089.165629:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234089.165630:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234089.165632:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234089.165633:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f7091f62. 02000000:00000001:3.0:1761234089.165635:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234089.165636:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234089.165637:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000e8dfec2c. 00000100:00000001:3.0:1761234089.165640:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234089.165641:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165643:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234089.165646:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234089.165647:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234089.165648:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234089.165651:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234086/real 1761234089] req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234089.165658:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234089.165659:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234089.165661:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234089.165663:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234089.165664:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234089.165667:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234089.165669:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234089.165676:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234089.165683:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234089.165684:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165686:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234089.165687:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234089.165689:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234089.165695:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234089.165698:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234089.165703:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234089.165705:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165706:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234089.165707:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165708:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234089.165710:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234089.165712:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234089.165714:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234089.165717:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234089.165719:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234089.165725:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032f5ca64 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228416:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234089.165729:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165731:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234089.165733:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000032f5ca64 x1846784245228416/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234111 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234089.165738:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234089.165739:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234089.165740:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000df51b12a. 02000000:00000001:3.0:1761234089.165744:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234089.165745:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234089.165746:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234089.165748:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234089.165749:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006ca8bb96. 02000000:00000001:3.0:1761234089.165753:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234089.165754:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234089.165755:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000032f5ca64. 00000100:00000001:3.0:1761234089.165758:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234089.165759:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165760:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234089.165770:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234089.165772:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234089.165774:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234089.165775:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234089.676930:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234089.676935:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234089.676937:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234089.676942:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234089.676947:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234089.676951:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234089.676953:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234089.676955:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234089.676958:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.060972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234090.060978:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234090.060980:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234090.060982:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.060987:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234090.060989:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.060991:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.060993:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.060996:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234090.060999:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234090.061000:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234090.061004:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234090.061005:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234090.061006:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.061008:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234090.061009:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.061010:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.061011:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.061014:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234090.061015:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234090.061015:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234090.061017:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234090.061017:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234090.061018:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.061019:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234090.061020:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.061021:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234090.061023:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234090.061173:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234090.061182:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234090.061189:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234090.700977:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234090.700983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234090.700984:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234090.700989:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234090.700994:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234090.700998:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234090.701000:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234090.701002:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234090.701005:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234091.085029:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234091.085035:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234091.085041:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234091.085042:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234091.085046:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234091.085047:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234091.085180:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234091.085184:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234091.085187:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234091.213067:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234091.213078:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234091.213079:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234091.213082:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234091.213084:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234091.213089:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234091.213095:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6560 00000100:00080000:3.0:1761234091.213098:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234091.213101:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213103:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234091.213105:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213106:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234091.213110:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234091.213111:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234091.213113:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234091.213116:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234091.213118:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234091.213121:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234091.213123:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213124:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234091.213126:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213127:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234091.213129:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234091.213131:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213133:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234091.213136:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234091.213138:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234091.213140:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213142:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234091.213144:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234091.213149:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f61de4c8. 00000020:00000040:3.0:1761234091.213152:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234091.213156:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234091.213157:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213159:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234091.213162:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234091.213165:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234091.213172:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000df51b12a. 00000100:00000001:3.0:1761234091.213177:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234091.213184:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234091.213205:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234091.213207:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234091.213210:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f61de4c8] to pc [ptlrpcd_rcv+-1] req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234091.213235:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213238:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234091.213240:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234091.213244:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234091.213246:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234091.213248:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234091.213249:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234091.213258:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234091.213262:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6560 00000100:00080000:3.0:1761234091.213265:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234091.213269:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213271:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1761234091.213305:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1761234091.213305:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213309:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:2.0:1761234091.213310:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1761234091.213313:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000100:00000001:2.0:1761234091.213314:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1761234091.213316:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234091.213319:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234091.213321:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000100:00000040:2.0:1761234091.213322:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000020:00000001:3.0:1761234091.213325:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234091.213330:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234091.213336:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234091.213337:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234091.213339:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213341:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1761234091.213345:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1761234091.213345:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 02000000:00000001:2.0:1761234091.213348:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213349:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00100000:2.0:1761234091.213354:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228480:192.168.201.124@tcp:250: 00000020:00000040:3.0:1761234091.213354:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234091.213360:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234091.213361:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1761234091.213365:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234091.213366:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234091.213368:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234091.213369:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234091.213373:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1761234091.213374:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234091.213379:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234091.213380:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000116ca75c. 00000100:00000001:3.0:1761234091.213381:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 02000000:00000001:2.0:1761234091.213384:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1761234091.213386:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234091.213390:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234091.213391:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00000001:3.0:1761234091.213392:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234091.213400:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002431276f. 00000400:00000010:2.0:1761234091.213402:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000020:00000040:3.0:1761234091.213404:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761234091.213409:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000200:2.0:1761234091.213411:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228480, portal 25 02000000:00000001:3.0:1761234091.213411:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213415:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234091.213417:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1761234091.213418:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1761234091.213420:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091124736 : -104717618426880 : ffffa0c2878d4c00) 02000000:00000001:3.0:1761234091.213422:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234091.213428:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234116 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 02000000:00000010:3.0:1761234091.213428:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000009baf651a. 00000100:00000001:3.0:1761234091.213434:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.213440:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00100000:3.0:1761234091.213442:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234091.213448:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234091.213452:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000001:3.0:1761234091.213455:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1761234091.213457:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245228480, offset 0 00000100:00000001:3.0:1761234091.213458:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234091.213463:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002431276f] to pc [ptlrpcd_rcv+-1] req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234091.213468:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213478:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213480:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234091.213482:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1761234091.213486:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:3.0:1761234091.213486:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234091.213489:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234091.213491:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234091.213492:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234091.213495:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000400:00000200:2.0:1761234091.213496:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00080000:3.0:1761234091.213499:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6560 00000400:00000200:2.0:1761234091.213503:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1761234091.213504:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1761234091.213509:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000040:3.0:1761234091.213512:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000200:2.0:1761234091.213517:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11787:11787:256:4294967295] 192.168.201.124@tcp LPNI seq info [11787:11787:8:4294967295] 00000100:00000001:3.0:1761234091.213517:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234091.213522:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213526:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234091.213529:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234091.213531:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761234091.213533:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000400:00000200:2.0:1761234091.213536:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000020:00000040:3.0:1761234091.213537:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:3.0:1761234091.213542:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:3.0:1761234091.213544:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000800:00000200:2.0:1761234091.213547:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234091.213554:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000100:00000040:3.0:1761234091.213554:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213559:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234091.213562:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213566:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000800:00000200:2.0:1761234091.213567:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000020:00000040:3.0:1761234091.213570:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000800:00000200:2.0:1761234091.213572:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000100:00080000:3.0:1761234091.213577:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213582:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761234091.213583:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000001:3.0:1761234091.213585:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.1:1761234091.213588:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 02000000:00000001:3.0:1761234091.213590:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234091.213594:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1761234091.213595:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000100:00000001:3.0:1761234091.213598:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000800:00000010:2.0:1761234091.213599:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000008:00100000:3.0:1761234091.213601:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1761234091.213606:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00000001:3.0:1761234091.213606:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213608:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234091.213614:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000100:00000010:3.0:1761234091.213614:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000108ab0a5. 00000020:00000040:3.0:1761234091.213619:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000200:2.0:1761234091.213621:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 02000000:00000001:3.0:1761234091.213622:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000400:00000010:2.0:1761234091.213625:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 02000000:00000001:3.0:1761234091.213625:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213629:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234091.213631:0:88993:0:(events.c:53:request_out_callback()) Process entered 02000000:00000001:3.0:1761234091.213631:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234091.213635:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234091.213636:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234116 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 02000000:00000010:3.0:1761234091.213640:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000071d5cb10. 00000100:00000001:3.0:1761234091.213645:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.213649:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00100000:3.0:1761234091.213653:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234091.213666:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234116 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234091.213681:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234091.213684:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234091.213684:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.213688:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000040:3.0:1761234091.213692:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000108ab0a5] to pc [ptlrpcd_rcv+-1] req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.213696:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.213700:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234091.213705:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.213709:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1761234091.213713:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234091.213715:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234091/real 1761234091] req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234091.213717:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:3.0:1761234091.213723:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6570) 00000400:00000001:2.0:1761234091.213727:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234091.213730:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234091.213731:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234091.213734:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234091.213735:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234091.213738:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234091.213741:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234091.213749:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.213756:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234091.213757:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.213760:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234091.213763:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234091.213766:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.213772:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234091.213776:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234091.213783:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234091.213786:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.213787:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234091.213789:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.213791:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234091.213793:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234091.213796:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234091.213799:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234091.213801:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234091.213805:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234091.213812:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228480:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234091.213824:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.213826:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234091.213828:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f61de4c8 x1846784245228480/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234091.213834:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234091.213837:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234091.213839:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000116ca75c. 02000000:00000001:2.0:1761234091.213843:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234091.213867:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234091.213869:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234091.213871:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234091.213874:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000df51b12a. 02000000:00000001:2.0:1761234091.213880:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234091.213882:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234091.213884:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f61de4c8. 00000100:00000001:2.0:1761234091.213889:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234091.213890:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.213893:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.213902:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234091.213903:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.213905:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234091.213906:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234091.213907:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234091.213910:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.213917:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234091.213919:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234091.213920:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234091.213922:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234091.213924:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228544:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234091.213928:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234091.213930:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234091.213931:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234091.213933:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234091.213936:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000116ca75c. 02000000:00000001:2.0:1761234091.213938:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234091.213941:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234091.213943:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234091.213946:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228544, portal 10 00000100:00000001:2.0:1761234091.213947:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234091.213948:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091128384 : -104717618423232 : ffffa0c2878d5a40) 00000100:00000040:2.0:1761234091.213954:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234116 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.213960:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234091.213963:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234091.213964:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234091.213966:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245228544, offset 0 00000400:00000200:2.0:1761234091.213970:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234091.213975:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234091.213979:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234091.213981:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234091.213984:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234091.213987:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11788:11788:256:4294967295] 192.168.201.124@tcp LPNI seq info [11788:11788:8:4294967295] 00000400:00000200:2.0:1761234091.213995:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234091.213999:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234091.214002:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234091.214006:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234091.214008:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761234091.214012:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234091.214014:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1761234091.214018:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234091.214020:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234091.214023:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234091.214026:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234091.214029:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234091.214030:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234091.214032:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234091.214035:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234116 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214042:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234091.214044:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234116 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214050:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.214052:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234091.214054:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.214056:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234091.214058:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.214060:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234091.214068:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234091/real 1761234091] req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234091.214075:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234091.214077:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234091.214078:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234091.214080:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234091.214081:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234091.214083:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234091.214085:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234091.214092:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214097:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234091.214099:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214101:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234091.214102:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234091.214104:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214110:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234091.214113:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234091.214119:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234091.214121:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214123:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234091.214124:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214125:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234091.214127:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234091.214129:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234091.214132:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234091.214134:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234091.214137:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234091.214143:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228544:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234091.214146:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234091.214148:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214153:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234091.214154:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234091.214156:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234091.214158:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234091.214160:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228608:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234091.214163:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234091.214164:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234091.214166:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234091.214168:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234091.214170:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234091.214172:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234091.214174:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234091.214177:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234091.214180:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228608, portal 4 00000100:00000001:2.0:1761234091.214181:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234091.214182:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091110144 : -104717618441472 : ffffa0c2878d1300) 00000100:00000040:2.0:1761234091.214195:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234116 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214202:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234091.214204:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234091.214206:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234091.214208:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245228608, offset 0 00000400:00000200:2.0:1761234091.214212:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234091.214217:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234091.214220:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234091.214223:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234091.214225:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234091.214227:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11789:11789:256:4294967295] 192.168.201.124@tcp LPNI seq info [11789:11789:8:4294967295] 00000400:00000200:2.0:1761234091.214234:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234091.214239:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234091.214241:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234091.214245:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234091.214247:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761234091.214251:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234091.214253:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1761234091.214257:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234091.214259:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234091.214261:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234091.214264:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234091.214266:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234091.214269:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234091.214271:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234091.214273:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234116 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214280:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234091.214283:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234116 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214303:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.214305:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234091.214307:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.214309:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234091.214311:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.214313:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234091.214316:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234091/real 1761234091] req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234091.214323:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234091.214324:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234091.214325:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234091.214327:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234091.214328:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234091.214330:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234091.214332:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234091.214339:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214344:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234091.214345:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214347:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234091.214348:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234091.214351:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234091.214356:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234091.214359:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234091.214365:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234091.214368:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214369:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234091.214370:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214371:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234091.214372:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234091.214375:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234091.214377:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234091.214379:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234091.214382:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234091.214389:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228608:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234091.214392:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214394:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234091.214396:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002431276f x1846784245228544/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234091.214402:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234091.214404:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234091.214405:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000116ca75c. 02000000:00000001:2.0:1761234091.214407:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234091.214408:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234091.214410:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234091.214411:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234091.214413:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000009baf651a. 02000000:00000001:2.0:1761234091.214416:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234091.214417:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234091.214418:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002431276f. 00000100:00000001:2.0:1761234091.214420:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234091.214421:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214423:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234091.214426:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000108ab0a5 x1846784245228608/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234116 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234091.214432:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234091.214433:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234091.214434:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234091.214436:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234091.214437:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234091.214437:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234091.214439:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234091.214440:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000071d5cb10. 02000000:00000001:2.0:1761234091.214442:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234091.214442:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234091.214443:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000108ab0a5. 00000100:00000001:2.0:1761234091.214446:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234091.214447:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214449:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.214452:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234091.214453:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.214455:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234091.214456:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234091.724905:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234091.724909:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234091.724910:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234091.724913:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234091.724917:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234091.724920:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.724921:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234091.724923:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234091.724925:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234092.108953:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234092.108957:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234092.108964:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234092.108965:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234092.108969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234092.108971:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234092.109049:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234092.109054:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234092.109058:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234092.749004:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234092.749011:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234092.749013:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234092.749020:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234092.749025:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234092.749029:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234092.749032:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234092.749034:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234092.749037:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234093.132989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234093.132994:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234093.133001:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234093.133002:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234093.133005:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234093.133006:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234093.133075:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234093.133077:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234093.133080:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234093.772965:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234093.772972:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234093.772975:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234093.772981:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234093.772987:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234093.772992:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234093.772994:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234093.772996:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234093.772998:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234094.157052:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234094.157058:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234094.157065:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234094.157067:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234094.157071:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234094.157073:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234094.157203:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234094.157208:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234094.157212:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234094.797018:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234094.797023:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234094.797025:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234094.797030:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234094.797036:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234094.797039:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234094.797042:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234094.797045:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234094.797047:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234095.181038:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234095.181045:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234095.181054:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234095.181056:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234095.181062:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234095.181064:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234095.181145:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234095.181150:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234095.181155:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234095.820926:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234095.820934:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234095.820936:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234095.820943:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234095.820949:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234095.820954:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234095.820958:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234095.820960:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234095.820964:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234096.205013:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234096.205019:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234096.205026:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234096.205027:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234096.205031:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234096.205031:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234096.205126:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234096.205130:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234096.205134:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234096.332998:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234096.333007:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234096.333008:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234096.333010:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234096.333012:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234096.333017:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234096.333022:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6565 00000100:00080000:1.0:1761234096.333024:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234096.333026:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333028:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234096.333030:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333031:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234096.333034:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234096.333035:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234096.333038:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234096.333041:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234096.333042:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234096.333044:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234096.333046:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333047:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234096.333048:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333048:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234096.333050:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234096.333052:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333053:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234096.333056:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234096.333058:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234096.333060:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333061:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234096.333062:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234096.333070:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761234096.333072:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234096.333076:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234096.333077:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333079:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234096.333081:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234096.333084:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234096.333091:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761234096.333095:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234096.333100:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234096.333107:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234096.333108:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234096.333111:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234096.333127:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333129:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234096.333130:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234096.333133:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234096.333133:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234096.333134:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234096.333135:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234096.333137:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234096.333138:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6565 00000100:00080000:1.0:1761234096.333139:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234096.333141:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333142:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234096.333143:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333145:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234096.333146:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234096.333148:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234096.333149:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234096.333150:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234096.333152:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234096.333153:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234096.333155:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333156:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234096.333157:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234096.333157:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234096.333160:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234096.333162:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00000001:2.0:1761234096.333162:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1761234096.333164:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333166:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333166:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234096.333169:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1761234096.333173:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:1.0:1761234096.333174:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234096.333177:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333180:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234096.333183:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1761234096.333185:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1761234096.333186:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333197:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234096.333201:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234096.333203:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333206:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234096.333208:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005940a194 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228672:192.168.201.124@tcp:250: 00000100:00000010:1.0:1761234096.333211:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000100:00000001:2.0:1761234096.333213:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1761234096.333214:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234096.333218:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234096.333218:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761234096.333220:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234096.333221:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333222:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234096.333224:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1761234096.333225:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234096.333227:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234096.333241:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 02000000:00000010:2.0:1761234096.333241:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234096.333245:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333247:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234096.333251:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00100000:1.0:1761234096.333253:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:2.0:1761234096.333258:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000001:1.0:1761234096.333259:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234096.333261:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234096.333264:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234096.333265:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228672, portal 25 00000100:00000001:2.0:1761234096.333270:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234096.333271:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039820992 : -104717669730624 : ffffa0c2847e76c0) 00000100:00000001:1.0:1761234096.333273:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333275:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234096.333277:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1761234096.333277:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234121 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234096.333282:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234096.333284:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234096.333285:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234096.333286:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234096.333289:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:1.0:1761234096.333290:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234096.333294:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6565 00000100:00000040:2.0:1761234096.333295:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00080000:1.0:1761234096.333296:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000010:2.0:1761234096.333298:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000040:1.0:1761234096.333300:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000200:2.0:1761234096.333302:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245228672, offset 0 00000100:00000001:1.0:1761234096.333303:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234096.333306:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333308:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000200:2.0:1761234096.333312:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000020:00000001:1.0:1761234096.333313:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234096.333315:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234096.333317:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234096.333318:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234096.333320:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234096.333321:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234096.333323:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333324:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1761234096.333327:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000040:1.0:1761234096.333328:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333331:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234096.333333:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234096.333335:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234096.333337:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234096.333338:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 02000000:00000001:1.0:1761234096.333339:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234096.333343:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1761234096.333343:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1761234096.333345:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333348:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000400:00000200:2.0:1761234096.333348:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000008:00100000:1.0:1761234096.333350:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234096.333354:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234096.333354:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11790:11790:256:4294967295] 192.168.201.124@tcp LPNI seq info [11790:11790:8:4294967295] 00000100:00000001:1.0:1761234096.333358:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234096.333363:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000020:00000040:1.0:1761234096.333365:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234096.333368:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234096.333369:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234096.333371:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:1.0:1761234096.333372:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234096.333373:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234096.333374:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234096.333378:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761234096.333381:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1761234096.333381:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234096.333386:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000100:00100000:1.0:1761234096.333387:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234096.333394:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234096.333395:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1761234096.333397:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000040:1.0:1761234096.333400:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:2.0:1761234096.333402:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1761234096.333406:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234096.333407:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234096.333410:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6575) 00000800:00000200:2.1:1761234096.333417:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234096.333419:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1761234096.333423:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234096.333426:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234096.333431:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234096.333435:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234096.333438:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234096.333440:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234096.333444:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234096.333446:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234121 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333452:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234096.333454:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234121 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333462:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333463:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234096.333467:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333469:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234096.333471:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333472:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234096.333474:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234096/real 1761234096] req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234096.333480:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234096.333481:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234096.333482:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234096.333483:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234096.333484:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234096.333485:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234096.333487:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234096.333492:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333495:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234096.333496:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333497:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234096.333499:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234096.333500:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333505:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234096.333507:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234096.333511:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234096.333513:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333514:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234096.333515:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333516:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234096.333517:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234096.333519:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234096.333521:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234096.333522:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234096.333524:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234096.333528:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005940a194 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228672:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234096.333531:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333532:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234096.333533:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000005940a194 x1846784245228672/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234096.333536:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234096.333538:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234096.333539:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234096.333542:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234096.333543:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234096.333543:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234096.333545:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234096.333546:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a3ed3058. 02000000:00000001:2.0:1761234096.333549:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234096.333550:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234096.333551:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000005940a194. 00000100:00000001:2.0:1761234096.333554:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234096.333555:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333556:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333562:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234096.333563:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333564:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234096.333565:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234096.333566:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234096.333567:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333571:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234096.333572:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234096.333573:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234096.333574:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234096.333576:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d6a5077 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228736:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234096.333577:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234096.333578:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234096.333579:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234096.333580:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234096.333582:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234096.333583:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234096.333585:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234096.333586:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234096.333588:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228736, portal 10 00000100:00000001:2.0:1761234096.333589:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234096.333590:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039794240 : -104717669757376 : ffffa0c2847e0e40) 00000100:00000040:2.0:1761234096.333593:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234121 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333597:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234096.333598:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234096.333599:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234096.333601:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245228736, offset 0 00000400:00000200:2.0:1761234096.333603:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234096.333606:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234096.333609:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234096.333610:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234096.333611:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234096.333613:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11791:11791:256:4294967295] 192.168.201.124@tcp LPNI seq info [11791:11791:8:4294967295] 00000400:00000200:2.0:1761234096.333618:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234096.333621:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234096.333623:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234096.333625:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234096.333626:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761234096.333629:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234096.333630:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1761234096.333633:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234096.333635:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234096.333636:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234096.333638:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234096.333640:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234096.333641:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234096.333642:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234096.333644:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234121 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333648:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234096.333649:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234121 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333652:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333653:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234096.333654:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333655:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234096.333656:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333658:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234096.333659:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234096/real 1761234096] req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234096.333663:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234096.333664:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234096.333665:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234096.333666:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234096.333667:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234096.333668:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234096.333669:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234096.333673:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333677:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234096.333677:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333678:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234096.333679:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234096.333680:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333683:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234096.333685:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234096.333688:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234096.333689:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333690:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234096.333691:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333692:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234096.333693:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234096.333694:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234096.333696:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234096.333697:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234096.333699:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234096.333702:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d6a5077 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228736:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234096.333706:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234096.333708:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333712:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234096.333712:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234096.333714:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234096.333714:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234096.333715:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000643267a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228800:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234096.333717:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234096.333718:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234096.333718:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234096.333720:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234096.333721:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000dd1dcb0e. 02000000:00000001:2.0:1761234096.333723:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234096.333724:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234096.333726:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234096.333727:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228800, portal 4 00000100:00000001:2.0:1761234096.333728:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234096.333729:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039800320 : -104717669751296 : ffffa0c2847e2600) 00000100:00000040:2.0:1761234096.333731:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234121 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333735:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234096.333736:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234096.333737:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234096.333738:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245228800, offset 0 00000400:00000200:2.0:1761234096.333740:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234096.333743:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234096.333744:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234096.333746:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234096.333747:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234096.333749:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11792:11792:256:4294967295] 192.168.201.124@tcp LPNI seq info [11792:11792:8:4294967295] 00000400:00000200:2.0:1761234096.333754:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234096.333757:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234096.333758:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234096.333761:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234096.333762:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761234096.333764:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234096.333766:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1761234096.333768:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234096.333768:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234096.333770:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234096.333771:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234096.333773:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234096.333774:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234096.333775:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234096.333776:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234121 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333780:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234096.333781:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234121 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333784:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333785:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234096.333786:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333787:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234096.333788:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333789:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234096.333790:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234096/real 1761234096] req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234096.333794:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234096.333794:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234096.333795:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234096.333796:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234096.333797:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234096.333798:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234096.333799:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234096.333807:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333811:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234096.333812:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333813:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234096.333813:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234096.333815:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234096.333818:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234096.333819:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234096.333823:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234096.333824:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333824:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234096.333825:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333826:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234096.333826:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234096.333828:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234096.333829:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234096.333830:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234096.333831:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234096.333835:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000643267a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228800:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234096.333837:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333838:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234096.333839:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007d6a5077 x1846784245228736/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234096.333871:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234096.333872:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234096.333873:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234096.333875:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234096.333876:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234096.333877:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234096.333878:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234096.333879:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:2.0:1761234096.333881:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234096.333881:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234096.333882:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007d6a5077. 00000100:00000001:2.0:1761234096.333884:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234096.333885:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333886:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234096.333887:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000643267a x1846784245228800/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234121 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234096.333891:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234096.333892:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234096.333892:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000dd1dcb0e. 02000000:00000001:2.0:1761234096.333894:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234096.333894:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234096.333895:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234096.333896:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234096.333896:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:2.0:1761234096.333897:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234096.333898:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234096.333898:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000643267a. 00000100:00000001:2.0:1761234096.333900:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234096.333900:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333901:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234096.333903:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234096.333904:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234096.333905:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234096.333905:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234096.844917:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234096.844924:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234096.844926:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234096.844931:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234096.844936:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234096.844939:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234096.844942:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234096.844944:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234096.844946:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234097.228935:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234097.228942:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234097.228949:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234097.228950:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234097.228953:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234097.228954:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234097.228993:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234097.228996:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234097.228998:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234097.868949:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234097.868958:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234097.868961:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234097.868969:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234097.868976:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234097.868982:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234097.868987:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234097.868990:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234097.868995:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234098.253001:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234098.253005:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234098.253010:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234098.253012:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234098.253014:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234098.253016:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234098.253108:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234098.253111:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234098.253113:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234098.892902:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234098.892908:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234098.892911:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234098.892916:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234098.892921:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234098.892924:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234098.892927:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234098.892929:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234098.892931:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234099.276925:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234099.276929:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234099.276934:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234099.276936:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234099.276938:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234099.276939:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234099.276964:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234099.276968:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234099.276971:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234099.916913:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234099.916921:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234099.916923:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234099.916929:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234099.916935:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234099.916941:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234099.916943:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234099.916946:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234099.916949:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.300979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234100.300986:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234100.300988:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234100.300990:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.300995:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234100.300997:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.300998:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.301000:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.301003:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234100.301004:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234100.301005:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234100.301008:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234100.301010:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234100.301010:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.301011:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234100.301012:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.301013:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.301014:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.301016:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234100.301017:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234100.301018:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234100.301019:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234100.301020:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234100.301020:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.301021:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234100.301023:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.301024:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234100.301025:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234100.301100:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234100.301104:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234100.301108:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234100.940905:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234100.940911:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234100.940913:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234100.940918:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234100.940924:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234100.940928:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234100.940930:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234100.940933:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234100.940935:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234101.324911:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234101.324915:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234101.324920:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234101.324921:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234101.324924:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234101.324924:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234101.324946:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234101.324950:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234101.324954:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234101.452958:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234101.452968:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234101.452970:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234101.452972:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234101.452975:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234101.452979:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234101.452983:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6570 00000100:00080000:1.0:1761234101.452986:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234101.452988:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234101.452990:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234101.452992:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234101.452993:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234101.452995:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234101.452996:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234101.452998:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234101.453001:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234101.453002:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234101.453004:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234101.453005:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234101.453006:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234101.453007:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234101.453008:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234101.453010:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234101.453013:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234101.453014:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234101.453017:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234101.453020:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234101.453021:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234101.453023:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234101.453024:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234101.453031:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761234101.453034:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234101.453037:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234101.453038:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234101.453040:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234101.453043:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234101.453046:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234101.453054:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761234101.453060:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234101.453067:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234101.453076:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234101.453078:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234101.453082:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234101.453135:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234101.453138:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234101.453140:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234101.453143:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234101.453144:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234101.453145:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234101.453146:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234101.453149:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234101.453151:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6570 00000100:00080000:1.0:1761234101.453153:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234101.453156:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234101.453158:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234101.453160:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234101.453162:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234101.453164:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234101.453166:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234101.453168:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234101.453170:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234101.453172:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234101.453174:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234101.453177:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234101.453180:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234101.453182:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234101.453184:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:2.0:1761234101.453184:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234101.453188:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1761234101.453189:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00000001:2.0:1761234101.453191:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1761234101.453195:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000040:2.0:1761234101.453198:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234101.453200:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234101.453205:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1761234101.453208:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234101.453209:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234101.453211:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234101.453213:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1761234101.453215:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234101.453217:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234101.453219:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234101.453223:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1761234101.453224:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228864:192.168.201.124@tcp:250: 00000008:00000001:1.0:1761234101.453228:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453231:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1761234101.453232:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234101.453236:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1761234101.453237:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 02000000:00000001:2.0:1761234101.453240:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1761234101.453241:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:2.0:1761234101.453244:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1761234101.453246:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234101.453249:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234101.453250:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:2.0:1761234101.453251:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000070e0ffb1. 02000000:00000001:1.0:1761234101.453252:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234101.453255:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234101.453256:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234101.453263:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 02000000:00000010:1.0:1761234101.453264:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761234101.453269:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234101.453272:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00100000:1.0:1761234101.453275:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234101.453281:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228864, portal 25 00000100:00000001:2.0:1761234101.453286:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234101.453287:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039795456 : -104717669756160 : ffffa0c2847e1300) 00000100:00000001:1.0:1761234101.453296:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1761234101.453296:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234126 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234101.453299:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234101.453307:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453311:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234101.453318:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234101.453322:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000001:1.0:1761234101.453325:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234101.453327:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245228864, offset 0 00000100:00000001:1.0:1761234101.453329:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234101.453333:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1761234101.453338:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234101.453340:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234101.453342:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234101.453344:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234101.453346:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234101.453348:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234101.453352:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6570 00000400:00000200:2.0:1761234101.453353:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00080000:1.0:1761234101.453358:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1761234101.453362:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000040:1.0:1761234101.453365:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000200:2.0:1761234101.453368:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1761234101.453370:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1761234101.453373:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000040:1.0:1761234101.453376:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000400:00000200:2.0:1761234101.453379:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11793:11793:256:4294967295] 192.168.201.124@tcp LPNI seq info [11793:11793:8:4294967295] 00000100:00000001:1.0:1761234101.453381:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234101.453386:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234101.453388:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000400:00000200:2.0:1761234101.453389:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000020:00000001:1.0:1761234101.453392:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234101.453395:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000800:00000200:2.0:1761234101.453396:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000020:00000001:1.0:1761234101.453399:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000800:00000010:2.0:1761234101.453401:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000020:00000001:1.0:1761234101.453404:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000800:00000200:2.0:1761234101.453409:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000040:1.0:1761234101.453412:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000800:00000200:2.0:1761234101.453413:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000100:00000001:1.0:1761234101.453417:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1761234101.453421:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000040:1.0:1761234101.453422:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000800:00000200:2.1:1761234101.453425:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000100:00000001:1.0:1761234101.453426:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234101.453430:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000800:00000100:2.0:1761234101.453431:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234101.453434:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000100:00080000:1.0:1761234101.453438:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000200:2.0:1761234101.453440:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1761234101.453441:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234101.453445:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1761234101.453446:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1761234101.453450:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1761234101.453450:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234101.453453:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 02000000:00000001:1.0:1761234101.453455:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453458:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000001:1.0:1761234101.453459:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000200:2.0:1761234101.453462:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234126 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000008:00100000:1.0:1761234101.453463:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234101.453469:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234101.453472:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453478:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000010:1.0:1761234101.453479:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000100:00000040:2.0:1761234101.453482:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234126 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000020:00000040:1.0:1761234101.453484:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234101.453489:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:2.0:1761234101.453491:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234101.453492:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453494:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761234101.453496:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234101.453498:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234101.453500:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453500:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453504:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1761234101.453507:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234101.453512:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453516:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234101.453517:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453519:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234101.453523:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234101/real 1761234101] req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234101.453545:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1761234101.453547:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1761234101.453548:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234101.453550:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000100:00000001:1.0:1761234101.453551:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234101.453553:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234101.453555:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000040:1.0:1761234101.453557:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453557:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234101.453562:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234101.453575:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234101.453578:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234101.453582:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6580) 00000100:00000200:2.0:1761234101.453582:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453590:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234101.453591:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453593:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234101.453595:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234101.453597:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453601:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234101.453604:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234101.453609:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234101.453611:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453612:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234101.453612:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453614:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234101.453616:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234101.453617:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234101.453619:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234101.453620:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234101.453622:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234101.453631:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245228864:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234101.453634:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453635:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234101.453636:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fd54b549 x1846784245228864/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234101.453640:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234101.453641:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234101.453643:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000070e0ffb1. 02000000:00000001:2.0:1761234101.453646:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234101.453647:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234101.453648:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234101.453649:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234101.453651:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f654abd8. 02000000:00000001:2.0:1761234101.453653:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234101.453654:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234101.453656:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fd54b549. 00000100:00000001:2.0:1761234101.453658:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234101.453658:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453660:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453665:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234101.453666:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453667:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234101.453668:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234101.453669:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234101.453671:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453675:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234101.453676:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234101.453677:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234101.453678:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234101.453680:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228928:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234101.453681:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234101.453683:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234101.453683:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234101.453685:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234101.453686:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000070e0ffb1. 02000000:00000001:2.0:1761234101.453687:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234101.453689:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234101.453691:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234101.453693:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228928, portal 10 00000100:00000001:2.0:1761234101.453694:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234101.453695:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039808832 : -104717669742784 : ffffa0c2847e4740) 00000100:00000040:2.0:1761234101.453698:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234126 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453702:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234101.453704:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234101.453705:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234101.453706:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245228928, offset 0 00000400:00000200:2.0:1761234101.453708:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234101.453711:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234101.453713:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234101.453715:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234101.453716:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234101.453718:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11794:11794:256:4294967295] 192.168.201.124@tcp LPNI seq info [11794:11794:8:4294967295] 00000400:00000200:2.0:1761234101.453723:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234101.453726:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234101.453728:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234101.453730:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234101.453731:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761234101.453734:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234101.453735:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1761234101.453737:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234101.453738:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234101.453740:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234101.453742:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234101.453744:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234101.453745:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234101.453746:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234101.453748:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234126 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453752:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234101.453753:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234126 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453757:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453758:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234101.453759:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453760:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234101.453761:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453762:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234101.453764:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234101/real 1761234101] req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234101.453768:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234101.453769:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234101.453769:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234101.453770:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234101.453771:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234101.453772:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234101.453773:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234101.453777:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453780:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234101.453781:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453782:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234101.453783:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234101.453784:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453788:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234101.453790:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234101.453794:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234101.453795:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453796:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234101.453797:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453798:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234101.453798:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234101.453800:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234101.453801:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234101.453802:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234101.453803:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234101.453807:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228928:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234101.453809:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234101.453810:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453814:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234101.453814:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234101.453815:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234101.453816:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234101.453817:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228992:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234101.453819:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234101.453820:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234101.453821:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234101.453822:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234101.453824:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000003562879b. 02000000:00000001:2.0:1761234101.453825:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234101.453826:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234101.453828:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234101.453829:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245228992, portal 4 00000100:00000001:2.0:1761234101.453830:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234101.453831:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039812480 : -104717669739136 : ffffa0c2847e5580) 00000100:00000040:2.0:1761234101.453834:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234126 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453837:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234101.453839:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234101.453840:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234101.453841:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245228992, offset 0 00000400:00000200:2.0:1761234101.453867:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234101.453871:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234101.453873:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234101.453874:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234101.453876:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234101.453877:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11795:11795:256:4294967295] 192.168.201.124@tcp LPNI seq info [11795:11795:8:4294967295] 00000400:00000200:2.0:1761234101.453882:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234101.453885:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234101.453886:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234101.453889:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234101.453890:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761234101.453893:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234101.453894:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1761234101.453897:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234101.453898:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234101.453900:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234101.453901:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234101.453903:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234101.453904:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234101.453905:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234101.453907:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234126 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453911:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234101.453913:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234126 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453917:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453918:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234101.453919:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453920:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234101.453921:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.453922:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234101.453924:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234101/real 1761234101] req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234101.453928:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234101.453929:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234101.453930:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234101.453931:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234101.453931:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234101.453932:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234101.453934:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234101.453937:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453941:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234101.453941:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453942:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234101.453943:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234101.453944:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234101.453948:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234101.453949:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234101.453952:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234101.453953:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453954:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234101.453955:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453956:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234101.453957:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234101.453958:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234101.453960:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234101.453961:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234101.453962:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234101.453966:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245228992:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234101.453968:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453969:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234101.453970:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f33bf85c x1846784245228928/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234101.453974:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234101.453974:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234101.453975:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000070e0ffb1. 02000000:00000001:2.0:1761234101.453977:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234101.453977:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234101.453978:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234101.453979:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234101.453980:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:2.0:1761234101.453982:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234101.453982:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234101.453983:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f33bf85c. 00000100:00000001:2.0:1761234101.453985:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234101.453985:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.453986:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234101.453987:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004567def3 x1846784245228992/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234126 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234101.453991:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234101.453992:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234101.453993:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000003562879b. 02000000:00000001:2.0:1761234101.453995:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234101.453995:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234101.453996:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234101.453997:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234101.453998:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:2.0:1761234101.453999:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234101.453999:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234101.454000:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004567def3. 00000100:00000001:2.0:1761234101.454001:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234101.454002:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.454003:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234101.454005:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234101.454005:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234101.454006:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234101.454007:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234101.964963:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234101.964969:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234101.964970:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234101.964976:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234101.964980:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234101.964984:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234101.964986:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234101.964989:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234101.964991:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234102.349042:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234102.349050:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234102.349059:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234102.349062:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234102.349068:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234102.349070:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234102.349223:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234102.349229:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234102.349235:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234102.989130:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234102.989139:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234102.989141:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234102.989148:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234102.989156:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234102.989161:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234102.989165:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234102.989168:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234102.989172:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234103.372925:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234103.372930:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234103.372937:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234103.372938:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234103.372942:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234103.372943:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234103.372988:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234103.372992:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234103.372995:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234104.013134:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234104.013159:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234104.013162:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234104.013169:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234104.013176:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234104.013181:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234104.013184:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234104.013188:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234104.013192:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234104.396971:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234104.396976:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234104.396981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234104.396982:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234104.396985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234104.396986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234104.397150:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234104.397155:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234104.397160:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234105.036927:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234105.036932:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234105.036933:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234105.036936:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234105.036940:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234105.036943:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234105.036945:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234105.036947:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234105.036948:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234105.420961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234105.420966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234105.420972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234105.420973:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234105.420977:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234105.420980:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234105.421031:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234105.421035:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234105.421038:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234106.060991:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234106.060997:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234106.060999:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234106.061025:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234106.061031:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234106.061036:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234106.061038:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234106.061041:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234106.061043:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234106.444973:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234106.444978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234106.444984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234106.444985:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234106.444987:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234106.444988:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234106.445063:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234106.445066:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234106.445069:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234106.572975:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234106.572984:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234106.572986:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234106.572987:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234106.572990:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234106.573008:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234106.573015:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6575 00000100:00080000:1.0:1761234106.573017:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234106.573019:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573021:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234106.573023:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573024:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234106.573027:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234106.573029:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234106.573031:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234106.573034:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234106.573035:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234106.573037:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234106.573039:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573041:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234106.573043:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573044:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234106.573046:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234106.573049:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573050:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234106.573053:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234106.573056:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234106.573058:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573059:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234106.573061:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234106.573068:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761234106.573070:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234106.573074:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234106.573075:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573077:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234106.573080:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234106.573082:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234106.573089:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761234106.573094:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234106.573101:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234106.573110:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234106.573112:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234106.573116:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234106.573134:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573136:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234106.573138:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234106.573141:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234106.573142:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234106.573143:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234106.573145:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234106.573147:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234106.573149:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6575 00000100:00080000:1.0:1761234106.573150:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234106.573153:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234106.573153:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234106.573156:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234106.573159:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234106.573159:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761234106.573162:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:2.0:1761234106.573162:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1761234106.573165:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234106.573166:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234106.573167:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234106.573169:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000100:00000040:2.0:1761234106.573169:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000020:00000001:1.0:1761234106.573172:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234106.573176:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234106.573182:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573185:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234106.573187:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234106.573187:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234106.573189:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573191:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 02000000:00000001:2.0:1761234106.573193:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1761234106.573194:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 02000000:00000001:2.0:1761234106.573196:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761234106.573197:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573200:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234106.573203:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1761234106.573203:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229056:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234106.573207:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761234106.573208:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234106.573211:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761234106.573212:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234106.573213:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573214:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 02000000:00000001:2.0:1761234106.573217:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1761234106.573218:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234106.573221:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234106.573222:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 00000100:00000001:1.0:1761234106.573223:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234106.573226:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234106.573228:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761234106.573232:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000400:00000010:2.0:1761234106.573233:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 02000000:00000001:1.0:1761234106.573235:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234106.573237:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573240:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234106.573249:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1761234106.573250:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 02000000:00000001:1.0:1761234106.573252:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234106.573258:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229056, portal 25 02000000:00000010:1.0:1761234106.573260:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:2.0:1761234106.573261:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234106.573263:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039807616 : -104717669744000 : ffffa0c2847e4280) 00000100:00000001:1.0:1761234106.573267:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234106.573268:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234131 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761234106.573275:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573276:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234106.573282:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234106.573285:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000001:1.0:1761234106.573288:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1761234106.573289:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245229056, offset 0 00000100:00000001:1.0:1761234106.573290:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234106.573295:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234106.573298:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573305:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573307:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234106.573308:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234106.573310:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234106.573311:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000400:00000200:2.0:1761234106.573313:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:1.0:1761234106.573314:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234106.573316:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234106.573319:0:468:0:(import.c:519:import_select_connection()) Process entered 00000400:00000200:2.0:1761234106.573321:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00080000:1.0:1761234106.573323:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6575 00000100:00080000:1.0:1761234106.573325:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1761234106.573328:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1761234106.573330:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573333:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1761234106.573334:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000040:1.0:1761234106.573337:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573341:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000200:2.0:1761234106.573341:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11796:11796:256:4294967295] 192.168.201.124@tcp LPNI seq info [11796:11796:8:4294967295] 00000020:00000001:1.0:1761234106.573343:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234106.573345:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234106.573347:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234106.573348:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234106.573350:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234106.573353:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000200:2.0:1761234106.573357:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000040:1.0:1761234106.573359:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573362:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234106.573364:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234106.573366:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234106.573368:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000800:00000200:2.0:1761234106.573377:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00080000:1.0:1761234106.573378:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000800:00000010:2.0:1761234106.573382:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000100:00000001:1.0:1761234106.573383:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234106.573387:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1761234106.573390:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000001:1.0:1761234106.573391:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1761234106.573395:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 02000000:00000001:1.0:1761234106.573396:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573401:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000800:00000200:2.1:1761234106.573403:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000008:00100000:1.0:1761234106.573404:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.1:1761234106.573407:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000008:00000001:1.0:1761234106.573409:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573412:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1761234106.573413:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234106.573416:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000100:00000010:1.0:1761234106.573418:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000400:00000200:2.0:1761234106.573420:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:1.0:1761234106.573423:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000200:2.0:1761234106.573427:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1761234106.573428:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234106.573431:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234106.573431:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234106.573434:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:1.0:1761234106.573435:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234106.573438:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234106.573440:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573441:0:88993:0:(events.c:53:request_out_callback()) Process entered 02000000:00000010:1.0:1761234106.573445:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000200:2.0:1761234106.573446:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234131 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234106.573450:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234106.573457:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573460:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000001:1.0:1761234106.573466:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1761234106.573466:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234131 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234106.573468:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234106.573476:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573479:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573482:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:1.0:1761234106.573487:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573488:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234106.573489:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000001:2.0:1761234106.573491:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234106.573494:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234106.573495:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6585) 00000100:00000001:2.0:1761234106.573496:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234106.573501:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234106/real 1761234106] req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234106.573508:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234106.573510:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234106.573511:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234106.573513:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234106.573514:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234106.573516:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234106.573518:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234106.573523:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573528:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234106.573530:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573531:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234106.573532:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234106.573534:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573538:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234106.573541:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234106.573545:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234106.573547:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573548:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234106.573549:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573550:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234106.573551:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234106.573553:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234106.573554:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234106.573556:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234106.573559:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234106.573566:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229056:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234106.573570:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573572:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234106.573575:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a014c997 x1846784245229056/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234106.573580:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234106.573583:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234106.573585:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234106.573588:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234106.573589:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234106.573591:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234106.573593:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234106.573595:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a174ebf2. 02000000:00000001:2.0:1761234106.573601:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234106.573602:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234106.573604:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a014c997. 00000100:00000001:2.0:1761234106.573607:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234106.573608:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573610:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573617:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234106.573619:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573621:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234106.573622:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234106.573623:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234106.573626:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573633:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234106.573634:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234106.573636:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234106.573637:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234106.573639:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229120:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234106.573641:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234106.573642:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234106.573643:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234106.573644:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234106.573646:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234106.573647:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234106.573649:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234106.573652:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234106.573653:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229120, portal 10 00000100:00000001:2.0:1761234106.573655:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234106.573655:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039814912 : -104717669736704 : ffffa0c2847e5f00) 00000100:00000040:2.0:1761234106.573659:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234131 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573663:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234106.573664:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234106.573665:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234106.573667:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245229120, offset 0 00000400:00000200:2.0:1761234106.573670:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234106.573673:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234106.573675:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234106.573677:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234106.573678:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234106.573680:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11797:11797:256:4294967295] 192.168.201.124@tcp LPNI seq info [11797:11797:8:4294967295] 00000400:00000200:2.0:1761234106.573686:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234106.573688:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234106.573690:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234106.573693:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234106.573694:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761234106.573697:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234106.573698:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1761234106.573700:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234106.573701:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234106.573703:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234106.573705:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234106.573707:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234106.573708:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234106.573709:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234106.573711:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234131 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573715:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234106.573717:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234131 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573721:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573722:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234106.573723:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573724:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234106.573725:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573726:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234106.573728:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234106/real 1761234106] req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234106.573732:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234106.573733:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234106.573734:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234106.573735:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234106.573736:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234106.573736:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234106.573738:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234106.573742:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573745:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234106.573746:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573747:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234106.573748:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234106.573749:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573753:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234106.573754:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234106.573758:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234106.573759:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573760:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234106.573761:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573762:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234106.573763:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234106.573764:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234106.573766:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234106.573767:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234106.573768:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234106.573772:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229120:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234106.573774:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234106.573776:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573779:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234106.573780:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234106.573781:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234106.573782:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234106.573783:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229184:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234106.573785:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234106.573786:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234106.573787:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234106.573788:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234106.573789:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234106.573790:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234106.573791:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234106.573793:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234106.573794:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229184, portal 4 00000100:00000001:2.0:1761234106.573795:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234106.573796:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039803968 : -104717669747648 : ffffa0c2847e3440) 00000100:00000040:2.0:1761234106.573799:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234131 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573802:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234106.573804:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234106.573805:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234106.573806:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245229184, offset 0 00000400:00000200:2.0:1761234106.573809:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234106.573811:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234106.573813:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234106.573815:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234106.573816:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234106.573828:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11798:11798:256:4294967295] 192.168.201.124@tcp LPNI seq info [11798:11798:8:4294967295] 00000400:00000200:2.0:1761234106.573834:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234106.573836:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234106.573838:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234106.573840:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234106.573842:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761234106.573870:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234106.573872:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1761234106.573876:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234106.573877:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234106.573880:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234106.573883:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234106.573886:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234106.573887:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234106.573889:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234106.573892:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234131 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573899:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234106.573901:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234131 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573907:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573908:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234106.573910:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573911:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234106.573912:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.573914:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234106.573915:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234106/real 1761234106] req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234106.573920:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234106.573921:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234106.573922:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234106.573923:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234106.573924:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234106.573925:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234106.573926:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234106.573930:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573934:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234106.573934:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573935:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234106.573936:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234106.573937:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234106.573941:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234106.573942:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234106.573946:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234106.573947:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573948:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234106.573949:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573950:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234106.573950:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234106.573952:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234106.573953:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234106.573954:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234106.573956:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234106.573960:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229184:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234106.573962:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573963:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234106.573964:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000632be758 x1846784245229120/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234106.573968:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234106.573968:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234106.573969:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234106.573971:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234106.573971:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234106.573972:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234106.573973:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234106.573974:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:2.0:1761234106.573975:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234106.573976:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234106.573976:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000632be758. 00000100:00000001:2.0:1761234106.573978:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234106.573979:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.573980:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234106.573981:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000bb0f086 x1846784245229184/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234131 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234106.573985:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234106.573985:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234106.573986:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234106.573987:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234106.573988:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234106.573988:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234106.573989:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234106.573990:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:2.0:1761234106.573991:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234106.573991:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234106.574006:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000bb0f086. 00000100:00000001:2.0:1761234106.574009:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234106.574010:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.574011:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234106.574013:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234106.574014:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234106.574015:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234106.574016:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234107.084950:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234107.084955:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234107.084957:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234107.084961:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234107.084965:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234107.084969:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234107.084972:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234107.084975:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234107.084977:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234107.469032:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234107.469037:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234107.469042:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234107.469043:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234107.469046:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234107.469048:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234107.469087:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234107.469090:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234107.469092:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234108.109044:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234108.109049:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234108.109051:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234108.109056:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234108.109062:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234108.109065:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234108.109068:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234108.109071:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234108.109074:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234108.492995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234108.492999:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234108.493004:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234108.493005:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234108.493007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234108.493008:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234108.493138:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234108.493142:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234108.493147:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234109.133023:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234109.133029:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234109.133031:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234109.133035:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234109.133041:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234109.133046:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234109.133048:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234109.133050:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234109.133053:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234109.516986:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234109.516990:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234109.516994:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234109.516995:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234109.516998:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234109.516999:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234109.517024:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234109.517027:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234109.517029:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234110.157072:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234110.157080:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234110.157082:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234110.157089:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234110.157095:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234110.157101:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234110.157103:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234110.157107:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234110.157110:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541010:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234110.541022:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234110.541026:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234110.541030:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541036:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234110.541038:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541040:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541042:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541048:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234110.541050:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234110.541051:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234110.541057:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234110.541059:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234110.541061:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541063:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234110.541064:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541065:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541066:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541069:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234110.541071:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234110.541071:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234110.541073:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234110.541074:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234110.541075:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541076:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234110.541077:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541078:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234110.541079:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234110.541181:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234110.541185:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234110.541188:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234111.180991:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234111.180999:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234111.181002:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234111.181008:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234111.181014:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234111.181019:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234111.181021:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234111.181024:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234111.181027:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234111.565007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234111.565012:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234111.565019:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234111.565020:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234111.565024:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234111.565025:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234111.565059:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234111.565064:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234111.565069:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234111.693048:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234111.693057:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234111.693058:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234111.693060:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234111.693062:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234111.693066:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234111.693072:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6580 00000100:00080000:3.0:1761234111.693074:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234111.693076:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693078:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234111.693079:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693081:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234111.693084:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234111.693086:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234111.693088:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234111.693090:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234111.693092:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234111.693094:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234111.693095:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693097:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234111.693098:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693099:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234111.693100:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234111.693103:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693104:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234111.693107:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234111.693109:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234111.693111:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693112:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234111.693114:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234111.693122:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000db341c88. 00000020:00000040:3.0:1761234111.693125:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234111.693128:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234111.693129:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693131:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234111.693133:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234111.693136:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234111.693142:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000001c1b0e94. 00000100:00000001:3.0:1761234111.693147:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234111.693152:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234111.693158:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234111.693160:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234111.693162:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000db341c88] to pc [ptlrpcd_rcv+-1] req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234111.693183:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693184:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234111.693185:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234111.693188:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234111.693188:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234111.693189:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234111.693190:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234111.693192:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234111.693193:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6580 00000100:00080000:3.0:1761234111.693194:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234111.693196:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693197:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234111.693197:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693199:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234111.693200:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234111.693201:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234111.693202:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234111.693203:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234111.693204:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234111.693205:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234111.693207:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693208:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234111.693209:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693209:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234111.693210:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234111.693212:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693213:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234111.693214:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234111.693218:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234111.693219:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693220:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234111.693222:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234111.693225:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693226:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234111.693228:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004b242e46. 00000020:00000040:3.0:1761234111.693230:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761234111.693231:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234111.693232:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693233:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234111.693233:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234111.693234:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234111.693237:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074003a94. 00000100:00000001:2.0:1761234111.693238:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234111.693239:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234111.693243:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693244:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234111.693247:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234111.693254:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234111.693255:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234111.693257:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234111.693262:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004b242e46] to pc [ptlrpcd_rcv+-1] req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693268:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234111.693271:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693272:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693275:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 02000000:00000001:2.0:1761234111.693276:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1761234111.693276:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1761234111.693279:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693279:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234111.693282:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234111.693283:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1761234111.693285:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db341c88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229248:192.168.201.124@tcp:250: 00000100:00080000:3.0:1761234111.693285:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234111.693289:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:2.0:1761234111.693291:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1761234111.693293:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6580 02000000:00000001:2.0:1761234111.693294:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1761234111.693295:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1761234111.693297:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234111.693298:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693302:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1761234111.693303:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1761234111.693304:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693306:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234111.693307:0:64357:0:(genops.c:899:class_conn2export()) Process entered 02000000:00000010:2.0:1761234111.693308:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 00000020:00000040:3.0:1761234111.693308:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761234111.693309:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234111.693311:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 02000000:00000001:2.0:1761234111.693312:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234111.693314:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:3.0:1761234111.693316:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:3.0:1761234111.693318:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000010:2.0:1761234111.693319:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00000001:3.0:1761234111.693320:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234111.693322:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693324:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234111.693325:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000010:2.0:1761234111.693326:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00080000:3.0:1761234111.693329:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693331:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234111.693332:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:2.0:1761234111.693333:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229248, portal 25 02000000:00000001:3.0:1761234111.693334:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234111.693336:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1761234111.693337:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693338:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091132032 : -104717618419584 : ffffa0c2878d6880) 00000100:00000001:3.0:1761234111.693339:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234111.693342:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234111.693344:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234111.693345:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234136 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234111.693346:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234111.693351:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000266eaa93. 00000020:00000040:3.0:1761234111.693353:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000001:2.0:1761234111.693356:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 02000000:00000001:3.0:1761234111.693356:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234111.693358:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693361:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:2.0:1761234111.693362:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 02000000:00000001:3.0:1761234111.693362:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234111.693364:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234111.693366:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 02000000:00000010:3.0:1761234111.693368:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074a27ae0. 00000100:00000200:2.0:1761234111.693370:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245229248, offset 0 00000100:00000001:3.0:1761234111.693371:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234111.693376:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234111.693380:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:3.0:1761234111.693385:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234111.693386:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234111.693388:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000266eaa93] to pc [ptlrpcd_rcv+-1] req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234111.693392:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234111.693393:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000200:2.0:1761234111.693395:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000040:3.0:1761234111.693396:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6590) 00000400:00000200:2.0:1761234111.693402:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234111.693406:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234111.693409:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234111.693412:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11799:11799:256:4294967295] 192.168.201.124@tcp LPNI seq info [11799:11799:8:4294967295] 00000400:00000200:2.0:1761234111.693420:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234111.693425:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234111.693429:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234111.693434:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234111.693436:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761234111.693440:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234111.693442:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1761234111.693444:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234111.693446:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234111.693451:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234111.693454:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234111.693457:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234111.693458:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234111.693462:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234111.693464:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234136 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693474:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234111.693475:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234136 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693480:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693481:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234111.693485:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693486:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234111.693488:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693489:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234111.693491:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234111/real 1761234111] req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234111.693496:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234111.693498:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234111.693499:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234111.693500:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234111.693501:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234111.693502:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234111.693503:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234111.693507:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693511:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234111.693512:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693514:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234111.693515:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234111.693517:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693521:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234111.693523:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234111.693527:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234111.693528:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693530:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234111.693530:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693532:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234111.693534:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234111.693535:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234111.693537:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234111.693538:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234111.693540:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234111.693545:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db341c88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229248:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234111.693547:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693548:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234111.693549:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000db341c88 x1846784245229248/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234111.693553:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234111.693555:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234111.693556:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234111.693558:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234111.693560:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234111.693560:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234111.693562:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234111.693563:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000001c1b0e94. 02000000:00000001:2.0:1761234111.693566:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234111.693567:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234111.693569:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000db341c88. 00000100:00000001:2.0:1761234111.693571:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234111.693571:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693572:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693579:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234111.693580:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693581:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234111.693582:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234111.693583:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234111.693584:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693588:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234111.693589:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234111.693591:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234111.693592:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234111.693593:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b242e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229312:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234111.693595:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234111.693595:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234111.693596:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234111.693597:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234111.693598:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234111.693599:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234111.693601:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234111.693602:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234111.693604:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229312, portal 10 00000100:00000001:2.0:1761234111.693605:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234111.693606:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091125952 : -104717618425664 : ffffa0c2878d50c0) 00000100:00000040:2.0:1761234111.693610:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234136 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693613:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234111.693615:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234111.693616:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234111.693617:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245229312, offset 0 00000400:00000200:2.0:1761234111.693619:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234111.693622:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234111.693624:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234111.693625:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234111.693627:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234111.693629:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11800:11800:256:4294967295] 192.168.201.124@tcp LPNI seq info [11800:11800:8:4294967295] 00000400:00000200:2.0:1761234111.693634:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234111.693636:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234111.693638:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234111.693640:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234111.693642:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761234111.693644:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234111.693646:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1761234111.693648:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234111.693649:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234111.693650:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234111.693652:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234111.693654:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234111.693655:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234111.693656:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234111.693657:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234136 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693661:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234111.693663:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234136 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693666:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693667:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234111.693668:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693669:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234111.693670:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693671:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234111.693672:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234111/real 1761234111] req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234111.693677:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234111.693677:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234111.693678:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234111.693679:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234111.693680:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234111.693681:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234111.693682:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234111.693686:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693693:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234111.693693:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693695:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234111.693695:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234111.693697:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693700:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234111.693702:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234111.693705:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234111.693706:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693707:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234111.693708:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693709:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234111.693709:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234111.693711:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234111.693712:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234111.693713:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234111.693714:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234111.693718:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b242e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229312:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234111.693720:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234111.693724:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693728:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234111.693729:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234111.693730:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234111.693731:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234111.693732:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000266eaa93 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229376:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234111.693734:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234111.693735:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234111.693736:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234111.693737:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234111.693738:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000070e0ffb1. 02000000:00000001:2.0:1761234111.693739:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234111.693741:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234111.693743:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234111.693744:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229376, portal 4 00000100:00000001:2.0:1761234111.693745:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234111.693746:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091129600 : -104717618422016 : ffffa0c2878d5f00) 00000100:00000040:2.0:1761234111.693748:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234136 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693752:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234111.693753:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234111.693754:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234111.693755:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245229376, offset 0 00000400:00000200:2.0:1761234111.693757:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234111.693759:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234111.693761:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234111.693762:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234111.693763:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234111.693765:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11801:11801:256:4294967295] 192.168.201.124@tcp LPNI seq info [11801:11801:8:4294967295] 00000400:00000200:2.0:1761234111.693769:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234111.693772:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234111.693774:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234111.693775:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234111.693777:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761234111.693779:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234111.693780:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1761234111.693782:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234111.693783:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234111.693785:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234111.693787:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234111.693788:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234111.693789:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234111.693790:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234111.693792:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234136 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693795:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234111.693796:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234136 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693800:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693800:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234111.693802:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693802:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234111.693803:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693804:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234111.693805:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234111/real 1761234111] req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234111.693809:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234111.693810:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234111.693810:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234111.693811:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234111.693812:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234111.693813:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234111.693814:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234111.693819:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693822:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234111.693822:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693823:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234111.693824:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234111.693825:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234111.693832:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234111.693834:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234111.693838:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234111.693839:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693840:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234111.693840:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693841:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234111.693842:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234111.693843:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234111.693868:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234111.693869:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234111.693871:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234111.693875:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000266eaa93 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229376:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234111.693877:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693878:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234111.693879:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004b242e46 x1846784245229312/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234111.693882:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234111.693883:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234111.693884:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234111.693886:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234111.693886:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234111.693887:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234111.693888:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234111.693889:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074003a94. 02000000:00000001:2.0:1761234111.693891:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234111.693891:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234111.693892:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004b242e46. 00000100:00000001:2.0:1761234111.693894:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234111.693895:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693913:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234111.693915:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000266eaa93 x1846784245229376/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234136 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234111.693919:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234111.693920:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234111.693921:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000070e0ffb1. 02000000:00000001:2.0:1761234111.693923:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234111.693924:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234111.693925:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234111.693926:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234111.693927:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074a27ae0. 02000000:00000001:2.0:1761234111.693928:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234111.693928:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234111.693929:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000266eaa93. 00000100:00000001:2.0:1761234111.693930:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234111.693931:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693932:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234111.693934:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234111.693934:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234111.693935:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234111.693936:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234112.205033:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234112.205037:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234112.205039:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234112.205042:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234112.205047:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234112.205049:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234112.205051:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234112.205054:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234112.205055:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234112.589006:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234112.589013:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234112.589022:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234112.589024:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234112.589028:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234112.589029:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234112.589136:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234112.589140:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234112.589146:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234113.229030:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234113.229041:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234113.229044:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234113.229053:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234113.229064:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234113.229071:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234113.229075:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234113.229079:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234113.229084:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234113.613077:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234113.613083:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234113.613089:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234113.613090:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234113.613093:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234113.613094:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234113.613144:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234113.613148:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234113.613153:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234114.253044:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234114.253051:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234114.253053:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234114.253060:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234114.253066:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234114.253071:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234114.253075:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234114.253078:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234114.253081:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234114.636959:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234114.636964:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234114.636970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234114.636971:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234114.636974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234114.636976:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234114.637049:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234114.637053:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234114.637056:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234115.277039:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234115.277045:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234115.277048:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234115.277053:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234115.277060:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234115.277064:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234115.277066:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234115.277069:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234115.277072:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234115.661019:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234115.661024:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234115.661031:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234115.661032:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234115.661038:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234115.661040:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234115.661161:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234115.661169:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234115.661177:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234116.301014:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234116.301021:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234116.301023:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234116.301029:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234116.301036:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234116.301040:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.301043:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234116.301046:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.301049:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234116.684966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234116.684971:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234116.684979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234116.684981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234116.684984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234116.684986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234116.685047:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234116.685050:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234116.685053:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761234116.813037:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234116.813067:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234116.813069:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:2.0:1761234116.813072:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234116.813075:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234116.813080:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234116.813087:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6585 00000100:00080000:2.0:1761234116.813091:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234116.813094:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813096:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234116.813099:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813101:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234116.813104:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234116.813106:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:2.0:1761234116.813109:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234116.813112:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:2.0:1761234116.813114:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:2.0:1761234116.813117:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:2.0:1761234116.813120:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813122:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234116.813123:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813125:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234116.813128:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:2.0:1761234116.813131:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813133:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234116.813136:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234116.813139:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234116.813142:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234116.813144:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234116.813146:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234116.813155:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000947ebee1. 00000020:00000040:2.0:1761234116.813159:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:2.0:1761234116.813163:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234116.813164:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234116.813168:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234116.813170:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234116.813174:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234116.813182:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078016eab. 00000100:00000001:2.0:1761234116.813189:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234116.813196:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234116.813208:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234116.813211:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234116.813217:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000947ebee1] to pc [ptlrpcd_rcv+-1] req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234116.813249:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234116.813253:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234116.813255:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234116.813261:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234116.813263:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1761234116.813265:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234116.813267:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234116.813271:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234116.813274:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6585 00000100:00080000:2.0:1761234116.813277:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234116.813281:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813284:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234116.813286:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813289:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234116.813292:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234116.813293:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:2.0:1761234116.813294:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234116.813296:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:2.0:1761234116.813298:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:2.0:1761234116.813300:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:2.0:1761234116.813302:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813304:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234116.813306:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813308:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234116.813327:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:2.0:1761234116.813332:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813336:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234116.813338:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234116.813345:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234116.813348:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234116.813351:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234116.813354:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1761234116.813356:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000008:00000001:2.0:1761234116.813358:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234116.813364:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.813365:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234116.813370:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000010:2.0:1761234116.813371:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000027d03ea6. 00000020:00000040:2.0:1761234116.813376:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000040:3.0:1761234116.813381:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234116.813384:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234116.813386:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234116.813392:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234116.813394:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234116.813397:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.813405:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 02000000:00000010:2.0:1761234116.813407:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000dd1dcb0e. 00000100:00000001:3.0:1761234116.813409:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234116.813416:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234116.813418:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234116.813424:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234116.813427:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761234116.813436:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000947ebee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229440:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234116.813451:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1761234116.813454:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1761234116.813457:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1761234116.813458:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234116.813463:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234116.813469:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000027d03ea6] to pc [ptlrpcd_rcv+-1] req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234116.813471:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234116.813480:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000be06bfbf. 02000000:00000001:3.0:1761234116.813485:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234116.813493:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234116.813495:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00000001:2.0:1761234116.813500:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234116.813504:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1761234116.813506:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000001:2.0:1761234116.813516:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00000200:3.0:1761234116.813518:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229440, portal 25 00000100:00080000:2.0:1761234116.813520:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234116.813525:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234116.813528:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1761234116.813530:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090834688 : -104717618716928 : ffffa0c28788df00) 00000100:00080000:2.0:1761234116.813533:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234116.813541:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000040:3.0:1761234116.813542:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234141 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761234116.813553:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6585 00000100:00000001:3.0:1761234116.813558:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00080000:2.0:1761234116.813563:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234116.813570:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234116.813574:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000040:2.0:1761234116.813576:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000200:3.0:1761234116.813581:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245229440, offset 0 00000100:00000001:2.0:1761234116.813584:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234116.813592:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000400:00000200:3.0:1761234116.813612:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813613:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234116.813619:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234116.813621:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:2.0:1761234116.813623:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234116.813627:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000400:00000200:3.0:1761234116.813630:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000020:00000001:2.0:1761234116.813632:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:2.0:1761234116.813637:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000200:3.0:1761234116.813639:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000040:2.0:1761234116.813644:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000200:3.0:1761234116.813646:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1761234116.813650:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1761234116.813651:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000040:2.0:1761234116.813658:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000400:00000200:3.0:1761234116.813660:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11802:11802:256:4294967295] 192.168.201.124@tcp LPNI seq info [11802:11802:8:4294967295] 00000100:00000001:2.0:1761234116.813664:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234116.813668:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:2.0:1761234116.813675:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234116.813680:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234116.813681:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 02000000:00000001:2.0:1761234116.813683:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234116.813706:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1761234116.813707:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 02000000:00000001:2.0:1761234116.813712:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1761234116.813716:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000100:00000001:2.0:1761234116.813717:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234116.813721:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234116.813725:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234116.813728:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1761234116.813729:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000010:2.0:1761234116.813736:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d95ed049. 00000800:00000200:3.0:1761234116.813736:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000020:00000040:2.0:1761234116.813742:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:2.0:1761234116.813747:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000800:00000200:3.1:1761234116.813748:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000001:2.0:1761234116.813749:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761234116.813754:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1761234116.813755:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234116.813759:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234116.813762:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1761234116.813764:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 02000000:00000010:2.0:1761234116.813769:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000062f3eded. 00000800:00000010:3.0:1761234116.813769:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234116.813775:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1761234116.813777:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234116.813789:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234116.813790:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1761234116.813840:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234116.813841:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234116.813845:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d95ed049] to pc [ptlrpcd_rcv+-1] req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234116.813856:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234116.813875:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234116.813878:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234116.813884:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234116.813889:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234141 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234116.813894:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:2.0:1761234116.813899:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6595) 00000100:00000001:3.0:1761234116.813902:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234116.813906:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234141 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.813913:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.813915:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234116.813922:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.813925:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234116.813927:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.813930:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234116.813933:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234116/real 1761234116] req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234116.813941:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234116.813943:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234116.813946:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234116.813948:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234116.813949:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234116.813950:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234116.813953:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234116.813960:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.813965:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234116.813967:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.813969:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234116.813971:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234116.813973:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.813978:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234116.813981:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234116.813987:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234116.813989:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.813990:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234116.813991:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.813993:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234116.813994:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234116.813996:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234116.813998:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234116.814000:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234116.814002:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234116.814013:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000947ebee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229440:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234116.814017:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814019:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234116.814020:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000947ebee1 x1846784245229440/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234116.814030:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234116.814032:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234116.814034:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000be06bfbf. 02000000:00000001:3.0:1761234116.814037:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234116.814038:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234116.814039:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234116.814042:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234116.814043:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078016eab. 02000000:00000001:3.0:1761234116.814046:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234116.814048:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234116.814049:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000947ebee1. 00000100:00000001:3.0:1761234116.814053:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234116.814054:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814056:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814063:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234116.814064:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.814066:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234116.814067:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234116.814068:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234116.814070:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814076:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234116.814077:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234116.814078:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234116.814080:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234116.814082:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000027d03ea6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229504:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234116.814084:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234116.814086:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234116.814087:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234116.814089:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234116.814091:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000be06bfbf. 02000000:00000001:3.0:1761234116.814093:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234116.814095:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234116.814097:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234116.814100:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229504, portal 10 00000100:00000001:3.0:1761234116.814101:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234116.814102:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090821312 : -104717618730304 : ffffa0c28788aac0) 00000100:00000040:3.0:1761234116.814106:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234141 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814111:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234116.814113:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234116.814115:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234116.814117:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245229504, offset 0 00000400:00000200:3.0:1761234116.814120:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234116.814124:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234116.814127:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234116.814129:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234116.814131:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234116.814133:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11803:11803:256:4294967295] 192.168.201.124@tcp LPNI seq info [11803:11803:8:4294967295] 00000400:00000200:3.0:1761234116.814140:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234116.814144:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234116.814146:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234116.814149:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234116.814151:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1761234116.814155:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234116.814156:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000100:3.0:1761234116.814159:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234116.814160:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234116.814163:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234116.814165:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234116.814168:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234116.814169:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234116.814171:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234116.814172:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234141 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814178:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234116.814180:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234141 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814184:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.814186:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234116.814188:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.814189:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234116.814190:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.814192:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234116.814193:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234116/real 1761234116] req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234116.814199:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234116.814200:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234116.814201:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234116.814202:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234116.814203:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234116.814204:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234116.814206:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234116.814211:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814216:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234116.814217:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814218:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234116.814219:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234116.814221:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814226:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234116.814228:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234116.814233:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234116.814236:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814237:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234116.814237:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814239:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234116.814240:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234116.814241:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234116.814243:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234116.814244:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234116.814246:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234116.814251:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000027d03ea6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229504:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234116.814254:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234116.814256:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814261:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234116.814262:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234116.814263:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234116.814264:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234116.814266:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d95ed049 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229568:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234116.814268:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234116.814269:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234116.814270:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234116.814271:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234116.814273:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062959873. 02000000:00000001:3.0:1761234116.814274:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234116.814276:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234116.814278:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234116.814280:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229568, portal 4 00000100:00000001:3.0:1761234116.814281:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234116.814282:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090823744 : -104717618727872 : ffffa0c28788b440) 00000100:00000040:3.0:1761234116.814285:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234141 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814290:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234116.814292:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234116.814294:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234116.814295:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245229568, offset 0 00000400:00000200:3.0:1761234116.814298:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234116.814301:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234116.814319:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234116.814322:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234116.814324:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234116.814326:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11804:11804:256:4294967295] 192.168.201.124@tcp LPNI seq info [11804:11804:8:4294967295] 00000400:00000200:3.0:1761234116.814334:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234116.814338:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234116.814341:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234116.814345:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234116.814346:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1761234116.814350:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234116.814352:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 33 secs later) 00000800:00000100:3.0:1761234116.814355:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234116.814357:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234116.814360:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234116.814363:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234116.814365:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234116.814367:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234116.814369:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234116.814371:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234141 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814378:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234116.814380:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234141 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814386:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.814388:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234116.814390:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.814391:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234116.814393:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.814395:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234116.814397:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234116/real 1761234116] req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234116.814404:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234116.814405:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234116.814406:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234116.814407:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234116.814409:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234116.814410:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234116.814412:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234116.814419:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814425:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234116.814426:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814427:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234116.814428:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234116.814431:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234116.814436:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234116.814439:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234116.814444:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234116.814446:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814448:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234116.814449:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814451:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234116.814452:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234116.814454:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234116.814457:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234116.814458:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234116.814461:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234116.814467:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d95ed049 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229568:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234116.814470:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814472:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234116.814474:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000027d03ea6 x1846784245229504/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234116.814479:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234116.814481:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234116.814482:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000be06bfbf. 02000000:00000001:3.0:1761234116.814484:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234116.814485:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234116.814486:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234116.814488:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234116.814489:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000dd1dcb0e. 02000000:00000001:3.0:1761234116.814491:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234116.814492:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234116.814493:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000027d03ea6. 00000100:00000001:3.0:1761234116.814496:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234116.814497:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814498:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234116.814500:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000d95ed049 x1846784245229568/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234141 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234116.814506:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234116.814507:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234116.814508:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062959873. 02000000:00000001:3.0:1761234116.814511:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234116.814513:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234116.814514:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234116.814516:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234116.814517:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062f3eded. 02000000:00000001:3.0:1761234116.814519:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234116.814520:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234116.814521:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000d95ed049. 00000100:00000001:3.0:1761234116.814525:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234116.814526:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814527:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234116.814530:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234116.814531:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234116.814533:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234116.814534:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234117.325063:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234117.325072:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234117.325075:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234117.325085:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234117.325094:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234117.325101:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234117.325104:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234117.325108:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234117.325114:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234117.708978:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234117.708983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234117.708988:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234117.708989:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234117.708991:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234117.708992:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234117.709091:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234117.709094:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234117.709096:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234118.348934:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234118.348939:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234118.348941:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234118.348945:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234118.348949:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234118.348952:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234118.348955:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234118.348957:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234118.348959:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234118.732930:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234118.732934:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234118.732939:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234118.732940:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234118.732942:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234118.732944:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234118.733012:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234118.733014:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234118.733016:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234119.373009:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234119.373016:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234119.373020:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234119.373025:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234119.373032:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234119.373036:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234119.373038:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234119.373041:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234119.373044:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234119.757017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234119.757023:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234119.757039:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234119.757040:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234119.757044:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234119.757045:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234119.757122:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234119.757127:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234119.757131:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234120.396965:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234120.396989:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234120.396991:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234120.396998:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234120.397005:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234120.397010:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234120.397013:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234120.397016:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234120.397019:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.780955:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234120.780961:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234120.780964:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234120.780966:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.780972:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234120.780974:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.780976:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.780978:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.780981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234120.780982:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234120.780983:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234120.780988:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234120.780990:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234120.780990:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.780992:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234120.780993:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.780995:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.780997:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.780999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234120.781000:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234120.781001:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234120.781003:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234120.781004:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234120.781005:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.781006:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234120.781006:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.781008:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234120.781009:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234120.781072:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234120.781075:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234120.781079:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234121.421009:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234121.421016:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234121.421017:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234121.421023:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234121.421029:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234121.421034:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234121.421036:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234121.421039:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234121.421042:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234121.805026:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234121.805033:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234121.805040:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234121.805043:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234121.805047:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234121.805048:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234121.805171:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234121.805175:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234121.805179:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234121.932965:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234121.932977:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234121.932979:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234121.932983:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234121.932986:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234121.932992:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234121.933000:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6590 00000100:00080000:3.0:1761234121.933003:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234121.933006:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933009:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234121.933011:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933013:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234121.933016:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234121.933018:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234121.933020:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234121.933023:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234121.933025:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234121.933027:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234121.933029:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933030:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234121.933031:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933033:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234121.933035:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234121.933037:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933039:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234121.933042:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234121.933044:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234121.933046:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234121.933048:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234121.933051:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234121.933061:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a52d2576. 00000020:00000040:3.0:1761234121.933064:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234121.933069:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234121.933070:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234121.933073:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234121.933075:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234121.933079:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234121.933088:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000062959873. 00000100:00000001:3.0:1761234121.933095:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234121.933103:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234121.933112:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234121.933114:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234121.933118:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a52d2576] to pc [ptlrpcd_rcv+-1] req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234121.933147:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234121.933150:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234121.933152:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234121.933158:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234121.933160:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234121.933162:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234121.933164:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234121.933168:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234121.933172:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6590 00000100:00000001:2.0:1761234121.933175:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1761234121.933175:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234121.933179:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234121.933182:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234121.933182:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1761234121.933197:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234121.933202:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933205:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234121.933207:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234121.933208:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000100:00000040:2.0:1761234121.933210:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000020:00000001:3.0:1761234121.933212:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234121.933214:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234121.933219:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234121.933223:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234121.933244:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933250:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234121.933257:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234121.933260:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234121.933261:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:2.0:1761234121.933263:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1761234121.933268:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 02000000:00000001:2.0:1761234121.933270:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234121.933274:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1761234121.933274:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933284:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234121.933288:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1761234121.933290:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229632:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234121.933297:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234121.933299:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234121.933301:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234121.933304:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234121.933304:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234121.933315:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 02000000:00000001:2.0:1761234121.933316:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1761234121.933324:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234121.933329:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234121.933330:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 00000100:00000001:3.0:1761234121.933340:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234121.933341:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234121.933357:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000194cd3a7. 00000400:00000010:2.0:1761234121.933359:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000020:00000040:3.0:1761234121.933367:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000400:00000010:2.0:1761234121.933370:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 02000000:00000001:3.0:1761234121.933372:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234121.933374:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234121.933377:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:2.0:1761234121.933378:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229632, portal 25 02000000:00000001:3.0:1761234121.933379:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234121.933382:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.933384:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234121.933387:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091112576 : -104717618439040 : ffffa0c2878d1c80) 02000000:00000010:3.0:1761234121.933392:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000be06bfbf. 00000100:00000040:2.0:1761234121.933397:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234146 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234121.933400:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.933411:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00100000:3.0:1761234121.933411:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234121.933425:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234121.933430:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000001:3.0:1761234121.933434:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1761234121.933435:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245229632, offset 0 00000100:00000001:3.0:1761234121.933437:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234121.933450:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000194cd3a7] to pc [ptlrpcd_rcv+-1] req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234121.933452:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933464:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234121.933466:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000200:2.0:1761234121.933469:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00080000:3.0:1761234121.933469:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234121.933474:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234121.933476:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000400:00000200:2.0:1761234121.933477:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000001:3.0:1761234121.933480:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1761234121.933483:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1761234121.933486:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1761234121.933487:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234121.933492:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11805:11805:256:4294967295] 192.168.201.124@tcp LPNI seq info [11805:11805:8:4294967295] 00000100:00000001:3.0:1761234121.933494:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234121.933503:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6590 00000400:00000200:2.0:1761234121.933516:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234121.933523:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00080000:3.0:1761234121.933539:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000800:00000010:2.0:1761234121.933542:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000100:00000040:3.0:1761234121.933548:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000800:00000200:2.0:1761234121.933552:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761234121.933554:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1761234121.933556:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000100:00000040:3.0:1761234121.933562:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000800:00000200:2.1:1761234121.933567:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761234121.933570:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000800:00000200:2.1:1761234121.933571:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1761234121.933579:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000020:00000001:3.0:1761234121.933579:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234121.933585:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000800:00000010:2.0:1761234121.933586:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000020:00000001:3.0:1761234121.933595:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000400:00000200:2.0:1761234121.933597:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:3.0:1761234121.933602:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000400:00000200:2.0:1761234121.933605:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1761234121.933608:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000200:2.0:1761234121.933611:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000020:00000001:3.0:1761234121.933613:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000010:2.0:1761234121.933615:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234121.933620:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000040:3.0:1761234121.933635:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000200:2.0:1761234121.933636:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234146 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234121.933640:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1761234121.933648:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234121.933652:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234146 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:3.0:1761234121.933655:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933662:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234121.933668:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:3.0:1761234121.933677:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234121.933681:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234121.933688:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1761234121.933692:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.933694:0:88993:0:(events.c:93:request_out_callback()) Process leaving 02000000:00000001:3.0:1761234121.933695:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234121.933698:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234121.933699:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.933716:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234121.933719:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234121.933721:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1761234121.933722:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1761234121.933724:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000008:00000001:3.0:1761234121.933728:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234121.933730:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234121.933731:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234121/real 1761234121] req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000010:3.0:1761234121.933735:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ecb9068. 00000020:00000040:3.0:1761234121.933740:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000001:2.0:1761234121.933744:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1761234121.933744:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000400:00000001:2.0:1761234121.933747:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234121.933761:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 02000000:00000001:3.0:1761234121.933764:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234121.933766:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000100:00000001:3.0:1761234121.933768:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1761234121.933769:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 02000000:00000001:3.0:1761234121.933772:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1761234121.933773:0:88993:0:(events.c:103:reply_in_callback()) Process entered 02000000:00000001:3.0:1761234121.933775:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234121.933779:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 02000000:00000010:3.0:1761234121.933782:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000296782d7. 00000100:00000001:3.0:1761234121.933788:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234121.933792:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761234121.933796:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.933822:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234121.933823:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.933826:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234121.933826:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234121.933832:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234121.933832:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234121.933849:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ecb9068] to pc [ptlrpcd_rcv+-1] req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761234121.933868:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.933879:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234121.933885:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234121.933893:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234121.933896:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.933897:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234121.933899:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.933903:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234121.933906:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234121.933910:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234121.933916:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234121.933922:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1761234121.933925:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234121.933931:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234121.933933:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00100000:2.0:1761234121.933940:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229632:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234121.933945:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.933947:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234121.933950:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a52d2576 x1846784245229632/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234121.933955:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 00000100:00000040:3.0:1761234121.933956:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6601) 02000000:00000001:2.0:1761234121.933958:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234121.933961:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234121.933965:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234121.933967:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234121.933969:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234121.933972:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234121.933974:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062959873. 02000000:00000001:2.0:1761234121.933979:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234121.933981:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234121.933982:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a52d2576. 00000100:00000001:2.0:1761234121.933986:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234121.933988:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.933990:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.933997:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234121.933999:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.934001:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234121.934003:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234121.934004:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234121.934006:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934011:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234121.934012:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234121.934014:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234121.934015:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234121.934017:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229696:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234121.934019:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234121.934021:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234121.934021:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234121.934023:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234121.934025:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234121.934026:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234121.934028:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234121.934030:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234121.934033:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229696, portal 10 00000100:00000001:2.0:1761234121.934034:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234121.934035:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091134464 : -104717618417152 : ffffa0c2878d7200) 00000100:00000040:2.0:1761234121.934039:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234146 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934043:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234121.934045:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234121.934046:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234121.934047:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245229696, offset 0 00000400:00000200:2.0:1761234121.934050:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234121.934054:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234121.934057:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234121.934059:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234121.934060:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234121.934062:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11806:11806:256:4294967295] 192.168.201.124@tcp LPNI seq info [11806:11806:8:4294967295] 00000400:00000200:2.0:1761234121.934068:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234121.934072:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234121.934074:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234121.934076:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234121.934078:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761234121.934082:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234121.934083:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1761234121.934086:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234121.934087:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234121.934089:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234121.934092:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234121.934094:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234121.934095:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234121.934096:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234121.934098:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234146 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934103:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234121.934104:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234146 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934108:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.934109:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234121.934111:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.934112:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234121.934113:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.934114:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234121.934116:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234121/real 1761234121] req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234121.934121:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234121.934122:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234121.934122:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234121.934123:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234121.934124:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234121.934125:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234121.934127:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234121.934131:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934135:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234121.934136:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934137:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234121.934138:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234121.934140:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934144:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234121.934146:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234121.934149:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234121.934151:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934151:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234121.934152:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934153:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234121.934154:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234121.934155:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234121.934157:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234121.934158:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234121.934160:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234121.934164:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229696:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234121.934167:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234121.934168:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934173:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234121.934174:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234121.934175:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234121.934176:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234121.934177:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229760:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234121.934179:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234121.934180:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234121.934182:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234121.934183:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234121.934184:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000116ca75c. 02000000:00000001:2.0:1761234121.934185:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234121.934187:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234121.934189:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234121.934190:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229760, portal 4 00000100:00000001:2.0:1761234121.934192:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234121.934192:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091118656 : -104717618432960 : ffffa0c2878d3440) 00000100:00000040:2.0:1761234121.934195:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234146 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934199:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234121.934201:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234121.934202:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234121.934203:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245229760, offset 0 00000400:00000200:2.0:1761234121.934206:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234121.934208:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234121.934210:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234121.934212:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234121.934213:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234121.934215:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11807:11807:256:4294967295] 192.168.201.124@tcp LPNI seq info [11807:11807:8:4294967295] 00000400:00000200:2.0:1761234121.934220:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234121.934223:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234121.934225:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234121.934227:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234121.934228:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761234121.934231:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234121.934232:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1761234121.934235:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234121.934236:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234121.934237:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234121.934240:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234121.934241:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234121.934243:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234121.934244:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234121.934245:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234146 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934250:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234121.934251:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234146 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934254:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.934255:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234121.934257:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.934257:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234121.934259:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.934259:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234121.934261:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234121/real 1761234121] req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234121.934265:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234121.934266:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234121.934267:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234121.934268:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234121.934269:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234121.934270:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234121.934271:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234121.934275:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934278:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234121.934279:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934280:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234121.934281:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234121.934282:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234121.934286:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234121.934287:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234121.934292:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234121.934293:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934294:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234121.934295:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934296:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234121.934297:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234121.934298:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234121.934299:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234121.934300:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234121.934302:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234121.934306:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229760:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234121.934308:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934309:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234121.934310:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000194cd3a7 x1846784245229696/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234121.934314:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234121.934315:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234121.934315:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234121.934317:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234121.934317:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234121.934318:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234121.934319:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234121.934320:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000be06bfbf. 02000000:00000001:2.0:1761234121.934322:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234121.934322:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234121.934323:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000194cd3a7. 00000100:00000001:2.0:1761234121.934325:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234121.934326:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934327:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234121.934328:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000009ecb9068 x1846784245229760/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234146 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234121.934332:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234121.934333:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234121.934334:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000116ca75c. 02000000:00000001:2.0:1761234121.934336:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234121.934336:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234121.934337:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234121.934338:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234121.934338:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000296782d7. 02000000:00000001:2.0:1761234121.934340:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234121.934341:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234121.934341:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000009ecb9068. 00000100:00000001:2.0:1761234121.934343:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234121.934344:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934345:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234121.934347:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234121.934347:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234121.934348:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234121.934349:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234122.444993:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234122.445002:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234122.445004:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234122.445010:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234122.445017:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234122.445022:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234122.445026:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234122.445030:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234122.445033:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234122.829023:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234122.829027:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234122.829033:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234122.829033:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234122.829036:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234122.829037:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234122.829100:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234122.829103:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234122.829105:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234123.468958:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234123.468963:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234123.468964:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234123.468969:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234123.468973:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234123.468977:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234123.468980:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234123.468983:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234123.468985:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234123.852970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234123.852974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234123.852979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234123.852980:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234123.852982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234123.852983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234123.853127:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234123.853133:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234123.853138:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234124.492972:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234124.492978:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234124.492979:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234124.492985:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234124.492991:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234124.492994:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234124.492997:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234124.493000:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234124.493002:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234124.876987:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234124.876991:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234124.876997:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234124.876998:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234124.877002:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234124.877004:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234124.877095:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234124.877099:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234124.877104:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234125.516973:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234125.516979:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234125.516981:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234125.516986:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234125.516991:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234125.516995:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234125.516998:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234125.517000:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234125.517002:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234125.900947:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234125.900952:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234125.900957:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234125.900959:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234125.900961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234125.900962:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234125.901050:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234125.901054:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234125.901056:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234126.540934:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234126.540940:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234126.540942:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234126.540947:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234126.540952:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234126.540956:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234126.540959:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234126.540961:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234126.540963:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234126.924917:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234126.924921:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234126.924927:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234126.924928:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234126.924931:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234126.924933:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234126.925022:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234126.925025:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234126.925028:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234127.052977:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234127.052988:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234127.052990:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234127.052993:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234127.052995:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234127.053000:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234127.053006:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6596 00000100:00080000:3.0:1761234127.053010:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234127.053013:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053015:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234127.053017:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053019:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234127.053023:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234127.053025:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234127.053027:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234127.053030:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234127.053032:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234127.053034:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234127.053037:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053039:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234127.053041:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053042:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234127.053044:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234127.053048:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053050:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234127.053053:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234127.053056:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234127.053058:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053060:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234127.053062:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234127.053072:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ecb9068. 00000020:00000040:3.0:1761234127.053075:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234127.053080:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234127.053081:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053084:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234127.053087:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234127.053090:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234127.053098:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000296782d7. 00000100:00000001:3.0:1761234127.053105:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234127.053111:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234127.053130:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234127.053132:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234127.053135:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ecb9068] to pc [ptlrpcd_rcv+-1] req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234127.053157:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053159:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234127.053161:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234127.053164:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234127.053165:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234127.053167:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234127.053168:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234127.053170:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234127.053172:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6596 00000100:00080000:3.0:1761234127.053175:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234127.053177:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053178:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234127.053180:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053182:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234127.053184:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234127.053185:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234127.053187:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234127.053189:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234127.053191:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234127.053192:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234127.053195:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053196:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234127.053198:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053200:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234127.053201:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234127.053204:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053206:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234127.053207:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234127.053213:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234127.053215:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053217:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234127.053221:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234127.053225:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053227:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053229:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1761234127.053231:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000194cd3a7. 00000100:00000001:2.0:1761234127.053235:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1761234127.053235:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:2.0:1761234127.053237:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1761234127.053240:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234127.053242:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234127.053244:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234127.053245:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234127.053247:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234127.053249:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234127.053254:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000be06bfbf. 00000100:00000001:2.0:1761234127.053255:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234127.053257:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053260:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234127.053261:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234127.053264:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234127.053269:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:2.0:1761234127.053270:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229824:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234127.053277:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234127.053280:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234127.053281:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053283:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1761234127.053284:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1761234127.053286:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234127.053290:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f7091f62. 02000000:00000001:2.0:1761234127.053293:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234127.053293:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000194cd3a7] to pc [ptlrpcd_rcv+-1] req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:2.0:1761234127.053300:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234127.053306:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234127.053313:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229824, portal 25 00000100:00000001:3.0:1761234127.053313:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053317:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000001:2.0:1761234127.053318:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1761234127.053319:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234127.053320:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091118656 : -104717618432960 : ffffa0c2878d3440) 00000100:00000001:3.0:1761234127.053326:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00000040:2.0:1761234127.053327:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234152 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761234127.053328:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234127.053332:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234127.053333:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234127.053338:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:2.0:1761234127.053342:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00080000:3.0:1761234127.053342:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6596 00000100:00000040:2.0:1761234127.053346:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00080000:3.0:1761234127.053346:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000010:2.0:1761234127.053349:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000040:3.0:1761234127.053350:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000200:2.0:1761234127.053352:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245229824, offset 0 00000100:00000001:3.0:1761234127.053354:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234127.053357:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053360:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000200:2.0:1761234127.053361:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000020:00000001:3.0:1761234127.053364:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234127.053366:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761234127.053367:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234127.053368:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:3.0:1761234127.053370:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:3.0:1761234127.053373:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000200:2.0:1761234127.053375:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000040:3.0:1761234127.053377:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053381:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1761234127.053385:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000040:3.0:1761234127.053386:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234127.053390:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000200:2.0:1761234127.053391:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1761234127.053393:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000200:2.0:1761234127.053396:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00080000:3.0:1761234127.053400:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000200:2.0:1761234127.053402:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11808:11808:256:4294967295] 192.168.201.124@tcp LPNI seq info [11808:11808:8:4294967295] 00000100:00000001:3.0:1761234127.053404:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234127.053408:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234127.053412:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234127.053415:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053418:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234127.053419:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1761234127.053420:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000008:00000001:3.0:1761234127.053423:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053426:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234127.053429:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a52d2576. 00000800:00000200:2.0:1761234127.053430:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000020:00000040:3.0:1761234127.053433:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000800:00000010:2.0:1761234127.053437:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 02000000:00000001:3.0:1761234127.053438:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234127.053440:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053442:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234127.053443:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234127.053445:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1761234127.053448:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000010:3.0:1761234127.053450:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000062959873. 00000800:00000200:2.0:1761234127.053453:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1761234127.053455:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234127.053463:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:2.1:1761234127.053465:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234127.053471:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1761234127.053475:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234127.053477:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1761234127.053481:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000100:00000040:3.0:1761234127.053484:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a52d2576] to pc [ptlrpcd_rcv+-1] req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000010:2.0:1761234127.053486:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234127.053494:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1761234127.053495:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.053499:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000200:2.0:1761234127.053503:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1761234127.053503:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6606) 00000400:00000200:2.0:1761234127.053509:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234127.053513:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234127.053519:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234127.053522:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234152 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053532:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234127.053535:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234152 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053541:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053543:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234127.053548:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053550:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234127.053552:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053554:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234127.053555:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234127/real 1761234127] req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234127.053562:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234127.053563:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234127.053564:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234127.053565:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234127.053566:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234127.053567:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234127.053569:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234127.053573:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053577:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234127.053578:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.053580:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234127.053581:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234127.053583:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053586:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234127.053589:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234127.053592:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234127.053594:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.053595:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234127.053595:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.053597:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234127.053598:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234127.053600:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234127.053601:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234127.053603:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234127.053633:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234127.053638:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245229824:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234127.053641:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.053643:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234127.053644:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000009ecb9068 x1846784245229824/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234127.053647:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234127.053649:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234127.053651:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f7091f62. 02000000:00000001:2.0:1761234127.053654:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234127.053655:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234127.053656:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234127.053657:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234127.053659:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000296782d7. 02000000:00000001:2.0:1761234127.053662:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234127.053663:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234127.053664:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000009ecb9068. 00000100:00000001:2.0:1761234127.053667:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234127.053667:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.053669:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.053675:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234127.053676:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053678:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234127.053679:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234127.053679:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234127.053681:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053685:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234127.053686:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234127.053687:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234127.053688:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234127.053690:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229888:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234127.053692:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234127.053693:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234127.053694:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234127.053695:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234127.053697:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000f7091f62. 02000000:00000001:2.0:1761234127.053698:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234127.053699:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234127.053701:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234127.053703:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229888, portal 10 00000100:00000001:2.0:1761234127.053704:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234127.053705:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091134464 : -104717618417152 : ffffa0c2878d7200) 00000100:00000040:2.0:1761234127.053709:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234152 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053713:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234127.053714:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234127.053716:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234127.053717:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245229888, offset 0 00000400:00000200:2.0:1761234127.053719:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234127.053723:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234127.053725:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234127.053727:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234127.053728:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234127.053730:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11809:11809:256:4294967295] 192.168.201.124@tcp LPNI seq info [11809:11809:8:4294967295] 00000400:00000200:2.0:1761234127.053736:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234127.053739:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234127.053740:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234127.053743:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234127.053744:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761234127.053747:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234127.053748:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1761234127.053750:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234127.053752:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234127.053753:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234127.053755:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234127.053757:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234127.053758:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234127.053759:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234127.053761:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234152 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053765:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234127.053766:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234152 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053770:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053770:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234127.053772:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053773:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234127.053774:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053775:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234127.053777:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234127/real 1761234127] req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234127.053781:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234127.053782:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234127.053782:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234127.053783:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234127.053784:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234127.053785:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234127.053786:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234127.053790:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053793:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234127.053794:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.053794:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234127.053795:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234127.053797:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053801:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234127.053802:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234127.053806:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234127.053807:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.053808:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234127.053808:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.053809:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234127.053810:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234127.053811:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234127.053813:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234127.053814:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234127.053815:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234127.053819:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229888:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234127.053821:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234127.053822:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053826:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234127.053826:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234127.053828:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234127.053829:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234127.053830:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229952:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234127.053832:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234127.053833:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234127.053833:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234127.053834:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234127.053836:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234127.053837:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234127.053839:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234127.053840:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234127.053842:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245229952, portal 4 00000100:00000001:2.0:1761234127.053883:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234127.053884:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091112576 : -104717618439040 : ffffa0c2878d1c80) 00000100:00000040:2.0:1761234127.053890:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234152 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053895:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234127.053897:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234127.053899:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234127.053901:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245229952, offset 0 00000400:00000200:2.0:1761234127.053904:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234127.053909:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234127.053912:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234127.053914:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234127.053916:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234127.053918:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11810:11810:256:4294967295] 192.168.201.124@tcp LPNI seq info [11810:11810:8:4294967295] 00000400:00000200:2.0:1761234127.053925:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234127.053928:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234127.053931:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234127.053934:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234127.053935:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761234127.053939:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234127.053940:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1761234127.053944:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234127.053945:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234127.053947:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234127.053950:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234127.053952:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234127.053953:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234127.053955:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234127.053956:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234152 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053962:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234127.053964:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234152 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.053969:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053970:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234127.053972:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053973:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234127.053974:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.053976:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234127.053977:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234127/real 1761234127] req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234127.053983:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234127.054001:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234127.054002:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234127.054003:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234127.054004:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234127.054005:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234127.054006:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234127.054010:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.054013:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234127.054014:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.054015:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234127.054016:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234127.054018:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234127.054021:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234127.054023:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234127.054026:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234127.054027:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.054028:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234127.054029:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.054030:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234127.054031:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234127.054032:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234127.054033:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234127.054034:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234127.054036:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234127.054040:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245229952:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234127.054042:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.054043:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234127.054044:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000194cd3a7 x1846784245229888/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234127.054047:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234127.054048:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234127.054049:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000f7091f62. 02000000:00000001:2.0:1761234127.054050:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234127.054051:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234127.054051:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234127.054052:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234127.054053:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000be06bfbf. 02000000:00000001:2.0:1761234127.054055:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234127.054056:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234127.054056:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000194cd3a7. 00000100:00000001:2.0:1761234127.054058:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234127.054058:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.054059:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234127.054060:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a52d2576 x1846784245229952/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234152 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234127.054064:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234127.054065:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234127.054065:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234127.054067:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234127.054067:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234127.054068:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234127.054069:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234127.054070:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062959873. 02000000:00000001:2.0:1761234127.054072:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234127.054072:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234127.054073:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a52d2576. 00000100:00000001:2.0:1761234127.054075:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234127.054075:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.054076:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234127.054078:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234127.054079:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234127.054080:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234127.054080:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234127.565000:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234127.565007:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234127.565009:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234127.565014:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234127.565020:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234127.565024:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234127.565026:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234127.565029:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234127.565031:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234127.948980:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234127.948985:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234127.948992:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234127.948994:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234127.948997:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234127.948999:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234127.949083:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234127.949086:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234127.949090:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234128.588925:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234128.588929:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234128.588930:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234128.588934:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234128.588937:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234128.588941:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234128.588942:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234128.588944:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234128.588946:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234128.972970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234128.972975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234128.972981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234128.972981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234128.972984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234128.972985:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234128.973083:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234128.973087:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234128.973091:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234129.612945:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234129.612950:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234129.612952:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234129.612956:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234129.612960:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234129.612963:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234129.612965:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234129.612966:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234129.612968:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.996973:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234129.996979:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234129.996981:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234129.996983:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.996988:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234129.996990:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.996992:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.996995:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.996999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234129.997001:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234129.997002:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234129.997007:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234129.997009:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234129.997010:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.997011:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234129.997012:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.997013:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.997014:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.997017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234129.997018:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234129.997019:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234129.997020:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234129.997021:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234129.997022:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.997023:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234129.997024:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.997025:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234129.997027:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234129.997108:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234129.997113:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234129.997117:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234130.636986:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234130.636992:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234130.636993:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234130.636998:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234130.637004:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234130.637007:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234130.637010:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234130.637012:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234130.637015:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234131.020971:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234131.020974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234131.020979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234131.020980:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234131.020983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234131.020984:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234131.021107:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234131.021111:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234131.021114:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234131.661009:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234131.661017:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234131.661019:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234131.661024:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234131.661029:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234131.661034:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234131.661036:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234131.661039:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234131.661042:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234132.044940:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234132.044944:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234132.044950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234132.044951:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234132.044954:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234132.044955:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234132.045051:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234132.045056:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234132.045060:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761234132.173024:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234132.173034:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234132.173036:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:2.0:1761234132.173053:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234132.173055:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234132.173059:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234132.173066:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6601 00000100:00080000:2.0:1761234132.173069:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234132.173071:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173074:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234132.173075:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173077:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234132.173079:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234132.173081:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:2.0:1761234132.173083:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234132.173085:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:2.0:1761234132.173087:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:2.0:1761234132.173088:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:2.0:1761234132.173090:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173091:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234132.173092:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173093:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234132.173095:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:2.0:1761234132.173098:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173100:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234132.173102:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234132.173105:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234132.173107:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173108:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234132.173109:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234132.173117:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d95ed049. 00000020:00000040:2.0:1761234132.173121:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:2.0:1761234132.173125:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234132.173126:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173129:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234132.173131:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234132.173135:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234132.173144:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078016eab. 00000100:00000001:2.0:1761234132.173151:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234132.173158:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234132.173168:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234132.173171:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234132.173175:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d95ed049] to pc [ptlrpcd_rcv+-1] req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234132.173203:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173205:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234132.173207:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234132.173211:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234132.173212:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1761234132.173214:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234132.173215:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234132.173218:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234132.173221:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6601 00000100:00080000:2.0:1761234132.173238:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234132.173240:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173242:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234132.173244:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173246:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234132.173248:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234132.173249:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:2.0:1761234132.173250:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234132.173251:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:2.0:1761234132.173254:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:2.0:1761234132.173256:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:2.0:1761234132.173258:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173260:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234132.173261:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173262:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234132.173264:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:2.0:1761234132.173266:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234132.173269:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234132.173270:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234132.173277:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1761234132.173280:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1761234132.173282:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234132.173287:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.173288:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1761234132.173294:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234132.173298:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234132.173301:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173303:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234132.173305:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000010:2.0:1761234132.173308:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000027d03ea6. 00000020:00000040:2.0:1761234132.173313:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:2.0:1761234132.173320:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234132.173321:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173328:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234132.173329:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234132.173331:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.173335:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 02000000:00000010:2.0:1761234132.173337:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000003562879b. 00000100:00000001:3.0:1761234132.173338:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234132.173344:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1761234132.173346:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234132.173348:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234132.173357:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d95ed049 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230016:192.168.201.124@tcp:250: 00000100:00100000:2.0:1761234132.173365:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.173366:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234132.173371:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234132.173374:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173377:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1761234132.173378:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1761234132.173384:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234132.173390:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000027d03ea6] to pc [ptlrpcd_rcv+-1] req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:3.0:1761234132.173391:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234132.173395:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234132.173405:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00000001:2.0:1761234132.173409:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173412:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234132.173415:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1761234132.173415:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000001:2.0:1761234132.173424:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234132.173425:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1761234132.173426:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234132.173428:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1761234132.173430:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230016, portal 25 00000100:00000001:2.0:1761234132.173432:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:3.0:1761234132.173435:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1761234132.173436:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6601 00000100:00000001:3.0:1761234132.173439:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090823744 : -104717618727872 : ffffa0c28788b440) 00000100:00080000:2.0:1761234132.173441:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234132.173448:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000040:3.0:1761234132.173452:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234157 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234132.173454:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234132.173460:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173467:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:3.0:1761234132.173487:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000001:2.0:1761234132.173489:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234132.173493:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:2.0:1761234132.173498:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000100:00000040:3.0:1761234132.173500:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000020:00000040:2.0:1761234132.173501:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000400:00000010:3.0:1761234132.173504:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000020:00000001:2.0:1761234132.173505:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:2.0:1761234132.173524:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234132.173529:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173531:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234132.173533:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000200:3.0:1761234132.173534:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245230016, offset 0 00000100:00000001:2.0:1761234132.173535:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234132.173568:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:2.0:1761234132.173571:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173574:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234132.173580:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1761234132.173581:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:2.0:1761234132.173585:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234132.173589:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173592:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234132.173593:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234132.173596:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173599:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234132.173603:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000947ebee1. 00000400:00000200:3.0:1761234132.173605:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000020:00000040:2.0:1761234132.173608:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:2.0:1761234132.173612:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234132.173614:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234132.173616:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234132.173620:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1761234132.173620:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 02000000:00000001:2.0:1761234132.173624:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234132.173629:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 02000000:00000010:2.0:1761234132.173630:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000cf0983ca. 00000400:00000200:3.0:1761234132.173636:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173637:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234132.173648:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234132.173648:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11811:11811:256:4294967295] 192.168.201.124@tcp LPNI seq info [11811:11811:8:4294967295] 00000100:00000001:2.0:1761234132.173668:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234132.173671:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234132.173678:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000040:2.0:1761234132.173684:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000947ebee1] to pc [ptlrpcd_rcv+-1] req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234132.173696:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1761234132.173698:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000001:2.0:1761234132.173700:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000010:3.0:1761234132.173704:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000100:00000040:2.0:1761234132.173705:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6611) 00000800:00000200:3.0:1761234132.173719:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234132.173723:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761234132.173732:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234132.173736:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1761234132.173740:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234132.173743:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234132.173749:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234132.173765:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234132.173770:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234132.173772:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234132.173777:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234132.173780:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234157 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.173789:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234132.173791:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234157 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.173796:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.173797:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234132.173805:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.173806:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234132.173809:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.173811:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234132.173813:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234132/real 1761234132] req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234132.173821:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234132.173823:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234132.173824:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234132.173826:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234132.173827:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234132.173828:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234132.173830:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234132.173834:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.173838:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234132.173839:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.173841:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234132.173843:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234132.173845:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.173849:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234132.173852:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234132.173857:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234132.173882:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.173883:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234132.173884:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.173887:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234132.173888:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234132.173891:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234132.173894:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234132.173896:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234132.173899:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234132.173905:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d95ed049 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230016:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234132.173909:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.173910:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234132.173912:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000d95ed049 x1846784245230016/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234132.173916:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234132.173918:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234132.173920:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234132.173923:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234132.173924:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234132.173926:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234132.173927:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234132.173929:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078016eab. 02000000:00000001:3.0:1761234132.173933:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234132.173934:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234132.173936:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000d95ed049. 00000100:00000001:3.0:1761234132.173941:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234132.173941:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.173943:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.173952:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234132.173953:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.173955:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234132.173957:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234132.173958:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234132.173960:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.173965:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234132.173966:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234132.173968:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234132.173969:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234132.173971:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000027d03ea6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230080:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234132.173973:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234132.173975:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234132.173976:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234132.173977:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234132.173979:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234132.173981:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234132.173983:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234132.173986:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234132.173988:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230080, portal 10 00000100:00000001:3.0:1761234132.173989:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234132.173990:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090821312 : -104717618730304 : ffffa0c28788aac0) 00000100:00000040:3.0:1761234132.173995:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234157 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.173999:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234132.174001:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234132.174002:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234132.174003:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245230080, offset 0 00000400:00000200:3.0:1761234132.174006:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234132.174010:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234132.174013:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234132.174014:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234132.174016:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234132.174018:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11812:11812:256:4294967295] 192.168.201.124@tcp LPNI seq info [11812:11812:8:4294967295] 00000400:00000200:3.0:1761234132.174024:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234132.174027:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234132.174029:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234132.174031:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234132.174033:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761234132.174036:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234132.174037:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1761234132.174040:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234132.174041:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234132.174043:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234132.174045:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234132.174048:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234132.174049:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234132.174050:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234132.174052:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234157 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174057:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234132.174058:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234157 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174062:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.174063:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234132.174065:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.174066:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234132.174067:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.174068:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234132.174070:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234132/real 1761234132] req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234132.174075:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234132.174076:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234132.174077:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234132.174078:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234132.174079:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234132.174080:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234132.174081:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234132.174086:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174090:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234132.174091:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174092:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234132.174093:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234132.174094:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174098:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234132.174100:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234132.174105:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234132.174106:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174107:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234132.174108:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174109:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234132.174110:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234132.174111:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234132.174113:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234132.174114:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234132.174116:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234132.174120:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000027d03ea6 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230080:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234132.174122:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234132.174124:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174129:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234132.174130:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234132.174131:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234132.174132:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234132.174133:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000947ebee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230144:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234132.174135:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234132.174136:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234132.174137:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234132.174138:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234132.174140:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234132.174141:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234132.174144:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234132.174146:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234132.174149:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230144, portal 4 00000100:00000001:3.0:1761234132.174151:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234132.174152:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090834688 : -104717618716928 : ffffa0c28788df00) 00000100:00000040:3.0:1761234132.174157:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234157 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174164:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234132.174167:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234132.174169:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234132.174172:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245230144, offset 0 00000400:00000200:3.0:1761234132.174176:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234132.174181:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234132.174185:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234132.174188:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234132.174190:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234132.174193:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11813:11813:256:4294967295] 192.168.201.124@tcp LPNI seq info [11813:11813:8:4294967295] 00000400:00000200:3.0:1761234132.174201:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234132.174206:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234132.174209:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234132.174213:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234132.174215:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761234132.174229:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234132.174232:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1761234132.174236:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234132.174238:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234132.174241:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234132.174244:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234132.174247:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234132.174249:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234132.174250:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234132.174253:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234157 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174261:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234132.174263:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234157 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174269:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.174271:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234132.174273:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.174275:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234132.174277:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.174278:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234132.174281:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234132/real 1761234132] req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234132.174288:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234132.174289:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234132.174291:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234132.174292:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234132.174294:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234132.174295:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234132.174298:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234132.174305:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174311:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234132.174312:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174314:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234132.174315:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234132.174318:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234132.174325:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234132.174328:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234132.174335:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234132.174337:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174338:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234132.174340:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174341:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234132.174343:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234132.174345:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234132.174347:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234132.174350:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234132.174353:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234132.174359:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000947ebee1 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230144:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234132.174363:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174365:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234132.174367:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000027d03ea6 x1846784245230080/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234132.174374:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234132.174375:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234132.174377:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234132.174380:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234132.174381:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234132.174382:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234132.174384:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234132.174385:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000003562879b. 02000000:00000001:3.0:1761234132.174389:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234132.174390:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234132.174391:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000027d03ea6. 00000100:00000001:3.0:1761234132.174394:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234132.174395:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174397:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234132.174399:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000947ebee1 x1846784245230144/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234157 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234132.174416:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234132.174418:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234132.174419:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234132.174421:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234132.174422:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234132.174423:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234132.174426:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234132.174427:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000cf0983ca. 02000000:00000001:3.0:1761234132.174430:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234132.174431:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234132.174432:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000947ebee1. 00000100:00000001:3.0:1761234132.174437:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234132.174438:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174440:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.174444:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234132.174445:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.174447:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234132.174448:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234132.684960:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234132.684966:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234132.684968:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234132.684974:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234132.684979:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234132.684983:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.684986:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234132.684988:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234132.684990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234133.069036:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234133.069041:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234133.069048:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234133.069049:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234133.069053:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234133.069054:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234133.069139:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234133.069143:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234133.069147:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234133.708991:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234133.708998:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234133.709000:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234133.709004:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234133.709009:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234133.709013:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234133.709016:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234133.709018:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234133.709021:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234134.093010:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234134.093015:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234134.093021:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234134.093022:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234134.093025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234134.093026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234134.093148:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234134.093154:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234134.093159:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234134.733076:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234134.733084:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234134.733086:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234134.733093:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234134.733100:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234134.733105:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234134.733108:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234134.733112:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234134.733115:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234135.116952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234135.116957:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234135.116962:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234135.116963:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234135.116966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234135.116967:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234135.117061:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234135.117064:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234135.117066:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234135.757045:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234135.757052:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234135.757054:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234135.757060:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234135.757067:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234135.757072:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234135.757075:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234135.757078:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234135.757081:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234136.140925:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234136.140929:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234136.140935:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234136.140937:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234136.140940:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234136.140941:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234136.141021:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234136.141024:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234136.141026:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234136.781014:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234136.781022:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234136.781025:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234136.781030:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234136.781038:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234136.781043:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234136.781047:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234136.781050:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234136.781054:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234137.165091:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234137.165103:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234137.165115:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234137.165117:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234137.165123:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234137.165125:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234137.165274:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234137.165279:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234137.165284:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761234137.292942:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234137.292956:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234137.292958:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:2.0:1761234137.292961:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234137.292964:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234137.292970:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234137.292978:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6606 00000100:00080000:2.0:1761234137.292982:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234137.292985:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.292988:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234137.292990:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.292992:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234137.292996:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234137.292998:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:2.0:1761234137.293001:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234137.293005:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:2.0:1761234137.293007:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:2.0:1761234137.293010:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:2.0:1761234137.293012:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293014:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234137.293015:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293017:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234137.293019:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:2.0:1761234137.293024:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293027:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234137.293030:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234137.293033:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234137.293036:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293038:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234137.293040:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234137.293052:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000032f5ca64. 00000020:00000040:2.0:1761234137.293056:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:2.0:1761234137.293061:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234137.293062:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293066:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234137.293069:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234137.293074:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234137.293086:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000001854c0bd. 00000100:00000001:2.0:1761234137.293093:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234137.293101:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234137.293111:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234137.293113:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234137.293117:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000032f5ca64] to pc [ptlrpcd_rcv+-1] req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234137.293137:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293139:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234137.293141:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234137.293144:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234137.293145:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1761234137.293146:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234137.293148:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234137.293150:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234137.293151:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6606 00000100:00080000:2.0:1761234137.293153:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234137.293155:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293157:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234137.293159:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293160:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234137.293162:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234137.293163:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:2.0:1761234137.293164:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234137.293166:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:2.0:1761234137.293167:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:2.0:1761234137.293168:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:2.0:1761234137.293170:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293172:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234137.293173:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293174:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234137.293175:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:2.0:1761234137.293177:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293178:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234137.293180:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234137.293186:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234137.293187:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293188:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234137.293192:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234137.293195:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293196:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234137.293199:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e8dfec2c. 00000020:00000040:2.0:1761234137.293200:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:2.0:1761234137.293203:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234137.293204:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293206:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234137.293207:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234137.293209:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234137.293212:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000070e0ffb1. 00000100:00000001:2.0:1761234137.293217:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234137.293222:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234137.293228:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234137.293229:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234137.293232:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e8dfec2c] to pc [ptlrpcd_rcv+-1] req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293236:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234137.293239:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293242:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234137.293243:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234137.293244:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234137.293247:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00000001:3.0:1761234137.293248:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1761234137.293250:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1761234137.293256:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234137.293258:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234137.293260:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234137.293268:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6606 00000100:00000040:3.0:1761234137.293269:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:2.0:1761234137.293272:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234137.293279:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293285:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234137.293290:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234137.293293:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234137.293294:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:3.0:1761234137.293296:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234137.293299:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234137.293302:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 02000000:00000001:3.0:1761234137.293303:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1761234137.293304:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234137.293307:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 02000000:00000001:3.0:1761234137.293308:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234137.293317:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00100000:3.0:1761234137.293321:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000032f5ca64 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230208:192.168.201.124@tcp:250: 00000020:00000001:2.0:1761234137.293323:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:3.0:1761234137.293333:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1761234137.293335:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293338:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1761234137.293339:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234137.293345:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234137.293348:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 02000000:00000001:3.0:1761234137.293351:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1761234137.293353:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234137.293358:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:2.0:1761234137.293361:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 02000000:00000010:3.0:1761234137.293362:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074003a94. 00000100:00000001:2.0:1761234137.293365:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234137.293366:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234137.293372:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234137.293376:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1761234137.293378:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 02000000:00000001:2.0:1761234137.293380:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293383:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234137.293384:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1761234137.293386:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000008:00000001:2.0:1761234137.293387:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293391:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234137.293395:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000598041cd. 00000100:00000200:3.0:1761234137.293395:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230208, portal 25 00000020:00000040:2.0:1761234137.293401:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000001:3.0:1761234137.293402:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234137.293403:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090812800 : -104717618738816 : ffffa0c287888980) 02000000:00000001:2.0:1761234137.293406:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234137.293429:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234137.293432:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234137.293433:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234137.293435:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234137.293438:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000cf0983ca. 00000100:00000040:3.0:1761234137.293442:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234162 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234137.293444:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234137.293454:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293469:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:2.0:1761234137.293484:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1761234137.293484:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293487:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234137.293489:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000040:2.0:1761234137.293493:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000598041cd] to pc [ptlrpcd_rcv+-1] req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234137.293494:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245230208, offset 0 00000100:00000001:2.0:1761234137.293507:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234137.293508:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:2.0:1761234137.293515:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:2.0:1761234137.293521:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6616) 00000400:00000200:3.0:1761234137.293531:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234137.293538:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234137.293542:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234137.293545:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234137.293548:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11814:11814:256:4294967295] 192.168.201.124@tcp LPNI seq info [11814:11814:8:4294967295] 00000400:00000200:3.0:1761234137.293558:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234137.293564:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234137.293568:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234137.293573:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234137.293576:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761234137.293582:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234137.293584:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1761234137.293588:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234137.293590:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234137.293595:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234137.293599:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234137.293603:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234137.293605:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234137.293609:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234137.293611:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234162 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293620:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234137.293621:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234162 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293627:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.293628:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234137.293633:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.293634:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234137.293636:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.293638:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234137.293641:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234137/real 1761234137] req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234137.293648:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234137.293650:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234137.293651:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234137.293652:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234137.293653:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234137.293655:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234137.293656:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234137.293666:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293671:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234137.293672:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293674:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234137.293675:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234137.293677:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293681:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234137.293684:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234137.293689:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234137.293691:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293692:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234137.293692:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293694:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234137.293696:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234137.293698:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234137.293701:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234137.293703:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234137.293705:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234137.293711:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000032f5ca64 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230208:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234137.293714:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234137.293716:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293721:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234137.293722:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234137.293723:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234137.293725:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234137.293726:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e8dfec2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230272:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234137.293729:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234137.293730:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234137.293731:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234137.293733:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234137.293735:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234137.293736:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234137.293738:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234137.293740:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234137.293741:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230272, portal 10 00000100:00000001:3.0:1761234137.293743:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234137.293744:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090837120 : -104717618714496 : ffffa0c28788e880) 00000100:00000040:3.0:1761234137.293748:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234162 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293752:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234137.293754:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234137.293755:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234137.293757:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245230272, offset 0 00000400:00000200:3.0:1761234137.293759:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234137.293763:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234137.293765:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234137.293766:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234137.293768:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234137.293770:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11815:11815:256:4294967295] 192.168.201.124@tcp LPNI seq info [11815:11815:8:4294967295] 00000400:00000200:3.0:1761234137.293776:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234137.293779:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234137.293780:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234137.293783:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234137.293784:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761234137.293787:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234137.293789:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1761234137.293791:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234137.293793:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234137.293795:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234137.293797:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234137.293799:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234137.293800:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234137.293801:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234137.293803:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234162 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293807:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234137.293809:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234162 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293813:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.293814:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234137.293816:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.293817:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234137.293818:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.293819:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234137.293821:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234137/real 1761234137] req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234137.293825:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234137.293826:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234137.293827:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234137.293828:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234137.293829:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234137.293830:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234137.293831:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234137.293836:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293840:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234137.293841:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293842:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234137.293877:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234137.293880:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293885:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234137.293887:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234137.293892:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234137.293893:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293894:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234137.293895:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293896:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234137.293897:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234137.293899:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234137.293901:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234137.293902:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234137.293904:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234137.293909:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e8dfec2c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230272:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234137.293912:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293913:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234137.293915:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000032f5ca64 x1846784245230208/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234137.293919:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234137.293921:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234137.293923:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234137.293925:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234137.293927:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234137.293928:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234137.293930:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234137.293931:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000001854c0bd. 02000000:00000001:3.0:1761234137.293935:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234137.293936:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234137.293937:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000032f5ca64. 00000100:00000001:3.0:1761234137.293941:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234137.293942:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293943:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234137.293944:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000e8dfec2c x1846784245230272/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234137.293949:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234137.293949:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234137.293950:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234137.293951:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234137.293952:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234137.293952:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234137.293953:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234137.293954:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000070e0ffb1. 02000000:00000001:3.0:1761234137.293957:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234137.293958:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234137.293959:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000e8dfec2c. 00000100:00000001:3.0:1761234137.293961:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234137.293962:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293963:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.293970:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234137.293972:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.293973:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234137.293974:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234137.293975:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234137.293977:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.293982:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234137.293983:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234137.293985:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234137.293986:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234137.293988:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000598041cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230336:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234137.293990:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234137.293991:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234137.293992:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234137.293994:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234137.293996:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234137.293997:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234137.293999:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234137.294001:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234137.294003:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230336, portal 4 00000100:00000001:3.0:1761234137.294004:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234137.294005:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090831040 : -104717618720576 : ffffa0c28788d0c0) 00000100:00000040:3.0:1761234137.294008:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234162 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.294012:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234137.294014:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234137.294016:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234137.294017:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245230336, offset 0 00000400:00000200:3.0:1761234137.294020:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234137.294023:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234137.294026:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234137.294028:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234137.294030:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234137.294032:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11816:11816:256:4294967295] 192.168.201.124@tcp LPNI seq info [11816:11816:8:4294967295] 00000400:00000200:3.0:1761234137.294037:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234137.294041:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234137.294043:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234137.294045:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234137.294047:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761234137.294050:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234137.294051:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1761234137.294054:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234137.294055:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234137.294057:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234137.294060:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234137.294061:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234137.294062:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234137.294064:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234137.294066:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234162 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.294071:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234137.294072:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234162 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.294076:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.294077:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234137.294079:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.294080:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234137.294082:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.294083:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234137.294085:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234137/real 1761234137] req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234137.294090:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234137.294091:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234137.294091:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234137.294092:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234137.294093:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234137.294094:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234137.294096:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234137.294101:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.294105:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234137.294106:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.294107:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234137.294108:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234137.294109:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234137.294113:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234137.294115:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234137.294124:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234137.294125:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.294126:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234137.294127:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.294129:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234137.294129:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234137.294131:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234137.294132:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234137.294133:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234137.294135:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234137.294139:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000598041cd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230336:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234137.294142:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.294143:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234137.294144:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000598041cd x1846784245230336/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234162 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234137.294148:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234137.294149:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234137.294149:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234137.294151:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234137.294152:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234137.294152:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234137.294153:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234137.294154:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000cf0983ca. 02000000:00000001:3.0:1761234137.294156:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234137.294157:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234137.294157:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000598041cd. 00000100:00000001:3.0:1761234137.294160:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234137.294161:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.294162:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.294164:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234137.294164:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.294166:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234137.294167:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234137.805009:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234137.805014:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234137.805016:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234137.805021:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234137.805026:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234137.805030:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.805033:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234137.805035:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234137.805037:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234138.188996:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234138.189003:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234138.189011:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234138.189012:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234138.189017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234138.189020:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234138.189152:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234138.189158:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234138.189163:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234138.828980:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234138.828987:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234138.828988:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234138.828993:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234138.828998:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234138.829002:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234138.829004:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234138.829006:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234138.829009:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234139.213000:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234139.213007:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234139.213016:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234139.213019:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234139.213025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234139.213027:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234139.213118:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234139.213123:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234139.213126:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234139.852958:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234139.852962:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234139.852963:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234139.852967:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234139.852971:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234139.852974:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234139.852976:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234139.852977:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234139.852979:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237051:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234140.237060:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234140.237063:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234140.237065:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237073:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234140.237076:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237079:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237082:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237088:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234140.237091:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234140.237093:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234140.237099:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234140.237101:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234140.237102:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237103:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234140.237104:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237105:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237107:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237111:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234140.237113:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234140.237115:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234140.237118:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234140.237120:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234140.237121:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237122:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234140.237123:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237124:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234140.237126:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234140.237228:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234140.237233:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234140.237238:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234140.876973:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234140.876979:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234140.876981:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234140.876986:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234140.876991:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234140.876994:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234140.876997:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234140.876999:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234140.877002:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234141.261025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234141.261032:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234141.261040:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234141.261041:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234141.261045:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234141.261046:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234141.261163:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234141.261167:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234141.261171:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234141.900972:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234141.900977:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234141.900978:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234141.900998:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234141.901003:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234141.901006:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234141.901008:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234141.901010:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234141.901012:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234142.285025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234142.285031:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234142.285037:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234142.285039:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234142.285043:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234142.285044:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234142.285120:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234142.285125:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234142.285128:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761234142.412993:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234142.413007:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234142.413009:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:2.0:1761234142.413013:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234142.413016:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234142.413023:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234142.413030:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6611 00000100:00080000:2.0:1761234142.413034:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234142.413037:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413039:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234142.413042:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413043:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234142.413047:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234142.413049:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:2.0:1761234142.413052:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234142.413055:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:2.0:1761234142.413057:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:2.0:1761234142.413059:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:2.0:1761234142.413061:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413062:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234142.413063:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413065:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234142.413067:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:2.0:1761234142.413069:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413073:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234142.413076:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234142.413079:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234142.413082:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413084:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234142.413086:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234142.413098:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcc2ec0d. 00000020:00000040:2.0:1761234142.413101:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:2.0:1761234142.413106:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234142.413107:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413110:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234142.413113:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234142.413118:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234142.413127:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000dd1dcb0e. 00000100:00000001:2.0:1761234142.413134:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234142.413141:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234142.413149:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234142.413152:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234142.413155:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcc2ec0d] to pc [ptlrpcd_rcv+-1] req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234142.413173:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413174:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234142.413176:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234142.413179:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234142.413180:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1761234142.413181:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234142.413182:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234142.413184:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234142.413185:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6611 00000100:00080000:2.0:1761234142.413187:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234142.413188:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413190:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234142.413191:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413193:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234142.413194:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234142.413195:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:2.0:1761234142.413196:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234142.413197:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:2.0:1761234142.413198:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:2.0:1761234142.413199:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:2.0:1761234142.413201:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413202:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234142.413203:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413204:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234142.413205:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:2.0:1761234142.413207:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413208:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234142.413209:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234142.413214:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234142.413215:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413217:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234142.413219:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234142.413222:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413223:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234142.413225:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000014a21c5a. 00000020:00000040:2.0:1761234142.413226:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:2.0:1761234142.413228:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234142.413228:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413229:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234142.413230:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234142.413231:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234142.413233:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f7091f62. 00000100:00000001:2.0:1761234142.413235:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234142.413238:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234142.413243:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234142.413244:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234142.413246:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000014a21c5a] to pc [ptlrpcd_rcv+-1] req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234142.413250:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413251:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234142.413252:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234142.413254:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234142.413255:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1761234142.413256:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234142.413257:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234142.413258:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234142.413260:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6611 00000100:00080000:2.0:1761234142.413261:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234142.413262:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413263:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234142.413265:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413267:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234142.413268:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234142.413268:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:2.0:1761234142.413269:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234142.413271:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:2.0:1761234142.413273:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:2.0:1761234142.413275:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234142.413278:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413280:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234142.413281:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413283:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234142.413285:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:2.0:1761234142.413287:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234142.413290:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234142.413291:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234142.413292:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234142.413293:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413294:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234142.413296:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234142.413297:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413298:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234142.413301:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e818ed53. 00000020:00000040:2.0:1761234142.413303:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000001:3.0:1761234142.413303:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1761234142.413309:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234142.413311:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413312:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234142.413327:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234142.413328:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.413330:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1761234142.413332:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000062f3eded. 00000100:00000001:3.0:1761234142.413335:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1761234142.413336:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234142.413344:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:3.0:1761234142.413353:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234142.413355:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234142.413356:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234142.413363:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e818ed53] to pc [ptlrpcd_rcv+-1] req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413400:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234142.413404:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413406:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234142.413408:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 02000000:00000001:3.0:1761234142.413409:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1761234142.413414:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6621) 02000000:00000001:3.0:1761234142.413414:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234142.413425:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcc2ec0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230400:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234142.413431:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234142.413435:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234142.413438:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234142.413441:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234142.413453:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234142.413455:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234142.413461:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234142.413467:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234142.413474:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230400, portal 25 00000100:00000001:3.0:1761234142.413476:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234142.413478:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090829824 : -104717618721792 : ffffa0c28788cc00) 00000100:00000040:3.0:1761234142.413483:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234167 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413491:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234142.413494:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234142.413496:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234142.413498:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245230400, offset 0 00000400:00000200:3.0:1761234142.413505:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234142.413520:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234142.413527:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234142.413531:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234142.413535:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234142.413538:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11817:11817:256:4294967295] 192.168.201.124@tcp LPNI seq info [11817:11817:8:4294967295] 00000400:00000200:3.0:1761234142.413548:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234142.413554:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234142.413558:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234142.413565:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234142.413568:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761234142.413574:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234142.413577:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1761234142.413581:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234142.413584:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234142.413589:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234142.413593:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234142.413597:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234142.413599:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234142.413604:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234142.413607:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234167 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413615:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234142.413618:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234167 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413623:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.413625:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234142.413631:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.413632:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234142.413635:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.413637:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234142.413640:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234142/real 1761234142] req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234142.413647:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234142.413649:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234142.413650:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234142.413652:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234142.413653:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234142.413655:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234142.413657:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234142.413664:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413669:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234142.413671:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.413673:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234142.413675:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234142.413677:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413683:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234142.413686:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234142.413691:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234142.413694:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.413695:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234142.413696:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.413699:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234142.413701:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234142.413704:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234142.413707:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234142.413709:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234142.413712:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234142.413718:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcc2ec0d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230400:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234142.413722:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234142.413724:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413729:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234142.413730:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234142.413732:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234142.413733:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234142.413735:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000014a21c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230464:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234142.413738:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234142.413740:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234142.413741:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234142.413743:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234142.413745:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234142.413747:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234142.413757:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234142.413760:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234142.413763:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230464, portal 10 00000100:00000001:3.0:1761234142.413764:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234142.413765:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090838336 : -104717618713280 : ffffa0c28788ed40) 00000100:00000040:3.0:1761234142.413770:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234167 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413779:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234142.413783:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234142.413786:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234142.413789:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245230464, offset 0 00000400:00000200:3.0:1761234142.413795:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234142.413801:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234142.413806:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234142.413809:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234142.413812:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234142.413816:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11818:11818:256:4294967295] 192.168.201.124@tcp LPNI seq info [11818:11818:8:4294967295] 00000400:00000200:3.0:1761234142.413826:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234142.413833:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234142.413837:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234142.413890:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234142.413894:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761234142.413901:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234142.413904:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1761234142.413909:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234142.413911:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234142.413915:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234142.413920:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234142.413923:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234142.413925:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234142.413927:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234142.413930:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234167 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413937:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234142.413939:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234167 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413945:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.413946:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234142.413950:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.413952:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234142.413954:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.413956:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234142.413958:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234142/real 1761234142] req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234142.413964:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234142.413965:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234142.413966:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234142.413968:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234142.413969:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234142.413971:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234142.413972:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234142.413978:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413983:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234142.413984:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.413986:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234142.413987:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234142.413989:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.413994:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234142.413996:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234142.414002:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234142.414004:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414005:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234142.414006:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414008:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234142.414009:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234142.414011:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234142.414014:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234142.414016:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234142.414018:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234142.414024:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000014a21c5a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230464:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234142.414028:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414029:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234142.414031:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fcc2ec0d x1846784245230400/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234142.414036:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234142.414038:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234142.414040:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234142.414043:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234142.414045:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234142.414047:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234142.414049:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234142.414051:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000dd1dcb0e. 02000000:00000001:3.0:1761234142.414054:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234142.414056:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234142.414057:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fcc2ec0d. 00000100:00000001:3.0:1761234142.414064:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234142.414065:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414066:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234142.414068:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000014a21c5a x1846784245230464/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234142.414073:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234142.414074:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234142.414075:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234142.414077:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234142.414078:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234142.414079:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234142.414080:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234142.414081:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f7091f62. 02000000:00000001:3.0:1761234142.414084:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234142.414085:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234142.414086:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000014a21c5a. 00000100:00000001:3.0:1761234142.414089:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234142.414090:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414092:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414100:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234142.414101:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.414104:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234142.414105:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234142.414106:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234142.414109:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.414115:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234142.414116:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234142.414118:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234142.414120:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234142.414122:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e818ed53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230528:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234142.414125:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234142.414126:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234142.414128:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234142.414129:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234142.414132:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234142.414133:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234142.414136:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234142.414138:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234142.414141:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230528, portal 4 00000100:00000001:3.0:1761234142.414142:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234142.414143:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090815232 : -104717618736384 : ffffa0c287889300) 00000100:00000040:3.0:1761234142.414148:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234167 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.414153:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234142.414156:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234142.414157:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234142.414159:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245230528, offset 0 00000400:00000200:3.0:1761234142.414163:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234142.414168:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234142.414171:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234142.414173:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234142.414175:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234142.414178:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11819:11819:256:4294967295] 192.168.201.124@tcp LPNI seq info [11819:11819:8:4294967295] 00000400:00000200:3.0:1761234142.414185:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234142.414189:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234142.414191:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234142.414195:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234142.414197:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1761234142.414201:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234142.414203:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1761234142.414206:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234142.414208:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234142.414210:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234142.414213:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234142.414215:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234142.414216:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234142.414218:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234142.414219:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234167 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.414225:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234142.414227:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234167 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.414232:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.414234:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234142.414236:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.414237:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234142.414239:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.414240:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234142.414242:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234142/real 1761234142] req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234142.414255:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234142.414256:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234142.414257:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234142.414258:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234142.414259:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234142.414260:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234142.414262:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234142.414268:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.414272:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234142.414273:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414275:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234142.414276:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234142.414278:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234142.414283:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234142.414285:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234142.414290:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234142.414292:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414293:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234142.414294:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414296:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234142.414297:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234142.414299:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234142.414301:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234142.414302:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234142.414305:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234142.414342:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e818ed53 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230528:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234142.414357:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414359:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234142.414361:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000e818ed53 x1846784245230528/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234167 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234142.414366:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234142.414367:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234142.414369:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234142.414371:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234142.414372:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234142.414373:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234142.414375:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234142.414376:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062f3eded. 02000000:00000001:3.0:1761234142.414379:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234142.414380:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234142.414381:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000e818ed53. 00000100:00000001:3.0:1761234142.414383:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234142.414384:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414385:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.414388:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234142.414390:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.414391:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234142.414393:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234142.924969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234142.924976:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234142.924977:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234142.924982:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234142.924987:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234142.924991:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.924993:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234142.924995:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234142.924998:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234143.309016:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234143.309024:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234143.309034:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234143.309035:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234143.309039:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234143.309042:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234143.309110:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234143.309114:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234143.309118:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234143.949009:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234143.949017:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234143.949019:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234143.949025:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234143.949032:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234143.949037:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234143.949041:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234143.949043:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234143.949047:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234144.333015:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234144.333022:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234144.333031:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234144.333032:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234144.333035:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234144.333037:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234144.333126:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234144.333131:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234144.333134:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234144.972973:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234144.972981:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234144.972983:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234144.972989:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234144.972996:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234144.973000:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234144.973003:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234144.973006:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234144.973009:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234145.356972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234145.356978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234145.356984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234145.356985:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234145.356989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234145.356989:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234145.357083:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234145.357087:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234145.357091:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234145.996977:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234145.996983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234145.996985:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234145.996990:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234145.996997:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234145.997000:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234145.997003:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234145.997005:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234145.997008:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234146.381026:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234146.381032:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234146.381040:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234146.381041:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234146.381045:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234146.381046:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234146.381170:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234146.381175:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234146.381179:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234147.021018:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234147.021025:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234147.021027:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234147.021032:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234147.021038:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234147.021042:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234147.021044:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234147.021047:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234147.021049:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234147.404983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234147.404988:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234147.404996:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234147.404997:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234147.405003:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234147.405003:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234147.405027:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234147.405031:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234147.405033:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761234147.533035:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234147.533052:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234147.533054:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:2.0:1761234147.533057:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234147.533061:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234147.533069:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234147.533077:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6616 00000100:00080000:2.0:1761234147.533082:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234147.533087:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533091:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234147.533094:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533097:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234147.533102:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234147.533105:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:2.0:1761234147.533108:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234147.533112:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:2.0:1761234147.533115:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:2.0:1761234147.533119:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:2.0:1761234147.533122:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533125:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234147.533128:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533129:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234147.533133:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:2.0:1761234147.533139:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533143:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234147.533146:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234147.533151:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234147.533155:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533158:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234147.533160:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234147.533172:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000069aeaefe. 00000020:00000040:2.0:1761234147.533176:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:2.0:1761234147.533181:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234147.533183:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533187:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234147.533190:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234147.533195:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234147.533205:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006ca8bb96. 00000100:00000001:2.0:1761234147.533213:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234147.533238:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234147.533250:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234147.533252:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234147.533257:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000069aeaefe] to pc [ptlrpcd_rcv+-1] req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234147.533278:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533280:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234147.533283:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234147.533287:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234147.533287:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1761234147.533289:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234147.533290:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234147.533293:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234147.533295:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6616 00000100:00080000:2.0:1761234147.533297:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234147.533301:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533304:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234147.533308:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533310:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234147.533312:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234147.533313:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:2.0:1761234147.533315:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234147.533317:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:2.0:1761234147.533319:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000001:3.0:1761234147.533319:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1761234147.533324:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:2.0:1761234147.533329:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234147.533329:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234147.533333:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1761234147.533334:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234147.533337:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533340:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234147.533343:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:2.0:1761234147.533349:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000040:3.0:1761234147.533351:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234147.533353:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234147.533356:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234147.533363:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234147.533370:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533375:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234147.533376:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234147.533380:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1761234147.533383:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1761234147.533385:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1761234147.533389:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234147.533390:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533394:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234147.533398:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000069aeaefe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230592:192.168.201.124@tcp:250: 00000100:00000010:2.0:1761234147.533401:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d82b1f3. 00000020:00000040:2.0:1761234147.533409:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:3.0:1761234147.533409:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234147.533416:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234147.533417:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234147.533418:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234147.533421:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533423:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234147.533427:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234147.533427:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1761234147.533433:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234147.533439:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000062f3eded. 02000000:00000010:3.0:1761234147.533440:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234147.533446:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533447:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234147.533453:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00100000:2.0:1761234147.533455:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:3.0:1761234147.533462:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000001:2.0:1761234147.533465:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234147.533468:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1761234147.533472:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230592, portal 25 00000100:00000040:2.0:1761234147.533473:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d82b1f3] to pc [ptlrpcd_rcv+-1] req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.533478:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234147.533480:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090833472 : -104717618718144 : ffffa0c28788da40) 00000100:00000040:3.0:1761234147.533492:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234172 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234147.533493:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533498:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234147.533499:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234147.533512:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234147.533513:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1761234147.533516:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234147.533517:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234147.533521:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:3.0:1761234147.533527:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00080000:2.0:1761234147.533530:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6616 00000100:00080000:2.0:1761234147.533535:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234147.533536:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000040:2.0:1761234147.533542:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000010:3.0:1761234147.533542:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000001:2.0:1761234147.533548:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1761234147.533550:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245230592, offset 0 00000100:00000040:2.0:1761234147.533554:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533558:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234147.533562:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234147.533563:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000400:00000200:3.0:1761234147.533566:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000020:00000001:2.0:1761234147.533567:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234147.533571:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:2.0:1761234147.533576:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:2.0:1761234147.533579:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234147.533583:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533586:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1761234147.533589:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000040:2.0:1761234147.533590:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533595:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234147.533597:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000200:3.0:1761234147.533602:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00080000:2.0:1761234147.533603:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234147.533609:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234147.533611:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1761234147.533613:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234147.533618:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1761234147.533619:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 02000000:00000001:2.0:1761234147.533624:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533629:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000400:00000200:3.0:1761234147.533630:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11820:11820:256:4294967295] 192.168.201.124@tcp LPNI seq info [11820:11820:8:4294967295] 00000008:00100000:2.0:1761234147.533633:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234147.533638:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234147.533640:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234147.533648:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000010:2.0:1761234147.533649:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d4d64bb. 00000020:00000040:2.0:1761234147.533654:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:2.0:1761234147.533662:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000800:00000200:3.0:1761234147.533663:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 02000000:00000001:2.0:1761234147.533669:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1761234147.533671:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000100:00000001:2.0:1761234147.533674:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234147.533677:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234147.533679:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1761234147.533684:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000010:2.0:1761234147.533686:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f7091f62. 00000800:00000200:3.0:1761234147.533690:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1761234147.533692:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234147.533699:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.1:1761234147.533713:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:2.0:1761234147.533716:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234147.533718:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761234147.533721:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000100:00000040:2.0:1761234147.533726:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d4d64bb] to pc [ptlrpcd_rcv+-1] req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000100:3.0:1761234147.533736:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234147.533740:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000100:00000001:2.0:1761234147.533749:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234147.533751:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1761234147.533752:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:2.0:1761234147.533759:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6626) 00000400:00000200:3.0:1761234147.533762:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234147.533774:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234147.533777:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234147.533781:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234147.533787:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234172 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.533798:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234147.533801:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234172 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.533808:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.533810:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234147.533816:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.533818:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234147.533820:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.533823:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234147.533826:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234147/real 1761234147] req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234147.533836:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234147.533838:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234147.533840:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234147.533842:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234147.533896:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234147.533899:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234147.533902:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234147.533910:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.533917:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234147.533919:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.533921:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234147.533922:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234147.533925:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.533929:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234147.533932:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234147.533936:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234147.533938:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.533938:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234147.533939:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.533941:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234147.533942:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234147.533944:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234147.533946:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234147.533948:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234147.533949:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234147.533954:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000069aeaefe pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230592:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234147.533957:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.533959:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234147.533960:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000069aeaefe x1846784245230592/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234147.533964:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234147.533966:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234147.533968:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234147.533970:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234147.533972:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234147.533973:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234147.533975:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234147.533976:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006ca8bb96. 02000000:00000001:3.0:1761234147.533980:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234147.533981:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234147.533982:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000069aeaefe. 00000100:00000001:3.0:1761234147.533985:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234147.533986:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.533987:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.533993:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234147.533994:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.533996:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234147.533997:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234147.533998:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234147.534000:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534004:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234147.534005:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234147.534007:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234147.534008:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234147.534009:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d82b1f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230656:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234147.534011:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234147.534013:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234147.534013:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234147.534015:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234147.534016:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234147.534018:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234147.534020:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234147.534022:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234147.534024:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230656, portal 10 00000100:00000001:3.0:1761234147.534025:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234147.534026:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090827392 : -104717618724224 : ffffa0c28788c280) 00000100:00000040:3.0:1761234147.534030:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234172 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534034:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234147.534036:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234147.534037:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234147.534038:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245230656, offset 0 00000400:00000200:3.0:1761234147.534041:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234147.534045:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234147.534047:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234147.534049:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234147.534050:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234147.534052:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11821:11821:256:4294967295] 192.168.201.124@tcp LPNI seq info [11821:11821:8:4294967295] 00000400:00000200:3.0:1761234147.534058:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234147.534061:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234147.534063:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234147.534065:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234147.534067:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761234147.534070:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234147.534071:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1761234147.534074:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234147.534075:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234147.534077:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234147.534079:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234147.534081:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234147.534082:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234147.534083:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234147.534084:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234172 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534089:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234147.534090:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234172 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534094:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.534095:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234147.534097:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.534098:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234147.534099:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.534100:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234147.534102:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234147/real 1761234147] req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234147.534106:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234147.534107:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234147.534108:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234147.534109:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234147.534109:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234147.534110:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234147.534112:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234147.534116:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534119:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234147.534120:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534120:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234147.534121:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234147.534122:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534126:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234147.534127:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234147.534131:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234147.534132:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534133:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234147.534134:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534135:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234147.534136:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234147.534137:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234147.534139:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234147.534140:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234147.534142:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234147.534146:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d82b1f3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230656:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234147.534148:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234147.534149:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534153:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234147.534153:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234147.534154:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234147.534155:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234147.534157:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d4d64bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230720:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234147.534158:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234147.534159:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234147.534160:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234147.534161:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234147.534163:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234147.534164:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234147.534165:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234147.534167:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234147.534168:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230720, portal 4 00000100:00000001:3.0:1761234147.534170:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234147.534170:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090818880 : -104717618732736 : ffffa0c28788a140) 00000100:00000040:3.0:1761234147.534173:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234172 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534176:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234147.534178:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234147.534179:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000200:3.0:1761234147.534180:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245230720, offset 0 00000400:00000200:3.0:1761234147.534183:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234147.534185:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234147.534187:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234147.534189:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234147.534190:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234147.534192:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11822:11822:256:4294967295] 192.168.201.124@tcp LPNI seq info [11822:11822:8:4294967295] 00000400:00000200:3.0:1761234147.534197:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234147.534199:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234147.534200:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 00000800:00000200:3.0:1761234147.534202:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234147.534204:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1761234147.534206:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234147.534207:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1761234147.534210:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234147.534211:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676282). 00000400:00000200:3.0:1761234147.534212:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234147.534214:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234147.534230:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:3.0:1761234147.534232:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:3.0:1761234147.534233:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234147.534235:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234172 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534240:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234147.534241:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234172 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534245:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.534245:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234147.534247:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.534248:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234147.534249:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.534251:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234147.534252:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234147/real 1761234147] req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234147.534257:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234147.534258:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234147.534258:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234147.534259:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234147.534260:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234147.534261:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234147.534263:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234147.534266:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534270:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234147.534270:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534271:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234147.534272:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234147.534273:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234147.534277:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234147.534278:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234147.534282:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234147.534283:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534284:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234147.534284:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534285:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234147.534287:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234147.534288:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234147.534289:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234147.534290:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234147.534292:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234147.534297:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d4d64bb pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230720:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234147.534299:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534300:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234147.534301:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007d82b1f3 x1846784245230656/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234147.534304:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234147.534305:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234147.534306:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234147.534307:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234147.534308:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234147.534308:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234147.534309:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234147.534310:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062f3eded. 02000000:00000001:3.0:1761234147.534312:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234147.534312:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234147.534313:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007d82b1f3. 00000100:00000001:3.0:1761234147.534316:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234147.534317:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534318:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234147.534319:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007d4d64bb x1846784245230720/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234172 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234147.534323:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234147.534323:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234147.534324:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234147.534325:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234147.534325:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234147.534326:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234147.534327:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234147.534327:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f7091f62. 02000000:00000001:3.0:1761234147.534328:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234147.534329:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234147.534330:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007d4d64bb. 00000100:00000001:3.0:1761234147.534331:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234147.534332:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534333:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234147.534335:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234147.534336:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234147.534337:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234147.534338:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234148.044929:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234148.044935:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234148.044936:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234148.044942:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234148.044947:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234148.044950:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234148.044952:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234148.044955:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234148.044957:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234148.428987:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234148.428993:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234148.429001:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234148.429002:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234148.429005:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234148.429006:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234148.429075:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234148.429079:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234148.429083:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234149.069016:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234149.069023:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234149.069025:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234149.069029:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234149.069035:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234149.069038:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234149.069040:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234149.069043:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234149.069046:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234149.452953:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234149.452958:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234149.452972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234149.452973:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234149.452976:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234149.452978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234149.453001:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234149.453007:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234149.453011:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234150.092916:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234150.092922:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234150.092924:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234150.092928:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234150.092933:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234150.092937:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234150.092939:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234150.092941:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234150.092943:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.476993:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234150.477001:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234150.477005:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234150.477008:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477015:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234150.477017:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477020:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477022:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234150.477027:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234150.477028:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234150.477034:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234150.477036:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234150.477037:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477038:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234150.477039:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477040:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477041:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477043:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234150.477045:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234150.477046:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234150.477048:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234150.477049:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234150.477050:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477051:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234150.477052:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477052:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234150.477054:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234150.477117:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234150.477121:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234150.477124:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234151.117003:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234151.117013:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234151.117015:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234151.117024:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234151.117031:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234151.117037:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234151.117040:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234151.117044:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234151.117048:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234151.500974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234151.500980:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234151.500989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234151.500991:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234151.500995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234151.500996:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234151.501019:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234151.501022:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234151.501025:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234152.141017:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234152.141023:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234152.141025:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234152.141029:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234152.141034:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234152.141038:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234152.141041:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234152.141043:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234152.141046:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234152.525024:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234152.525030:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234152.525038:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234152.525040:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234152.525043:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234152.525044:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234152.525175:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234152.525180:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234152.525184:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234152.652979:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234152.652996:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234152.652998:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234152.653001:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234152.653004:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234152.653013:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234152.653022:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6621 00000100:00080000:1.0:1761234152.653026:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234152.653031:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653034:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234152.653037:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653040:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234152.653045:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234152.653047:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234152.653050:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234152.653053:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234152.653056:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234152.653059:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234152.653062:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653064:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234152.653065:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653067:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234152.653070:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234152.653074:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653077:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234152.653081:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234152.653084:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234152.653087:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234152.653089:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234152.653092:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234152.653104:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761234152.653108:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234152.653114:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234152.653115:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234152.653134:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234152.653137:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234152.653142:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234152.653153:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761234152.653162:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234152.653170:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234152.653181:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234152.653184:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234152.653188:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234152.653209:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234152.653212:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234152.653214:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234152.653217:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234152.653219:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234152.653220:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234152.653221:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234152.653224:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234152.653226:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6621 00000100:00080000:1.0:1761234152.653228:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234152.653231:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653233:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234152.653235:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653237:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234152.653239:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234152.653240:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234152.653241:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234152.653243:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234152.653245:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234152.653247:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234152.653250:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653251:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234152.653253:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653255:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234152.653257:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234152.653261:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653264:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234152.653266:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234152.653275:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234152.653278:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653278:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234152.653281:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234152.653284:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000008:00100000:1.0:1761234152.653286:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1761234152.653287:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234152.653293:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000008:00000001:1.0:1761234152.653297:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234152.653301:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653303:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234152.653306:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234152.653309:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 02000000:00000001:3.0:1761234152.653310:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1761234152.653314:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761234152.653314:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234152.653320:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00100000:3.0:1761234152.653320:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002b682c08 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230784:192.168.201.124@tcp:250: 02000000:00000001:1.0:1761234152.653323:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653327:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1761234152.653328:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234152.653330:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234152.653332:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761234152.653333:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234152.653334:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234152.653338:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1761234152.653340:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 02000000:00000010:3.0:1761234152.653343:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074a27ae0. 00000100:00000001:1.0:1761234152.653346:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234152.653347:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234152.653355:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:3.0:1761234152.653356:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234152.653365:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000001:1.0:1761234152.653366:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234152.653367:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234152.653373:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234152.653374:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230784, portal 25 00000100:00000001:3.0:1761234152.653380:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234152.653381:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039801536 : -104717669750080 : ffffa0c2847e2ac0) 00000100:00000001:1.0:1761234152.653387:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234152.653389:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234177 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234152.653391:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234152.653394:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234152.653397:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:1.0:1761234152.653400:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00000040:3.0:1761234152.653401:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00080000:1.0:1761234152.653403:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000400:00000010:3.0:1761234152.653404:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000cfe59242. 00000100:00000001:1.0:1761234152.653406:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:3.0:1761234152.653408:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245230784, offset 0 00000100:00080000:1.0:1761234152.653409:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234152.653414:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000400:00000200:3.0:1761234152.653417:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00080000:1.0:1761234152.653420:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6621 00000100:00080000:1.0:1761234152.653423:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234152.653426:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653429:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234152.653431:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653433:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234152.653436:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000400:00000200:3.0:1761234152.653436:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000020:00000040:1.0:1761234152.653439:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234152.653442:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234152.653444:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000400:00000200:3.0:1761234152.653446:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000020:00000001:1.0:1761234152.653447:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234152.653451:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000200:3.0:1761234152.653451:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234152.653455:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000040:1.0:1761234152.653459:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000200:3.0:1761234152.653462:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11823:11823:256:4294967295] 192.168.201.124@tcp LPNI seq info [11823:11823:8:4294967295] 00000100:00000001:1.0:1761234152.653464:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234152.653468:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653471:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234152.653475:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000200:3.0:1761234152.653476:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00080000:1.0:1761234152.653481:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000800:00000200:3.0:1761234152.653485:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234152.653486:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234152.653489:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:3.0:1761234152.653491:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000d8a943a (tot 1676530). 02000000:00000001:1.0:1761234152.653492:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234152.653496:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234152.653498:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000800:00000200:3.0:1761234152.653500:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000008:00100000:1.0:1761234152.653501:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234152.653505:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1761234152.653506:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:1.0:1761234152.653509:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234152.653515:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000020:00000040:1.0:1761234152.653517:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234152.653519:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234152.653519:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234152.653521:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234152.653523:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234152.653524:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653528:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234152.653529:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:3.0:1761234152.653532:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1761234152.653534:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653536:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234152.653540:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234152.653540:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653543:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761234152.653550:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234152.653551:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234152.653553:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234152.653554:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234152.653556:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1761234152.653558:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653560:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234152.653561:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000001:3.0:1761234152.653562:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234152.653564:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761234152.653566:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6631) 00000100:00000040:3.0:1761234152.653568:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234152.653577:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234152.653579:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234152.653582:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234152.653584:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234152.653587:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230848:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234152.653589:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234152.653590:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234152.653591:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234152.653592:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234152.653595:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234152.653597:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234152.653599:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000009aaa4f6b. 00000400:00000010:3.0:1761234152.653601:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000a390468f. 00000100:00000200:3.0:1761234152.653603:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230848, portal 10 00000100:00000001:3.0:1761234152.653604:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234152.653605:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039803968 : -104717669747648 : ffffa0c2847e3440) 00000100:00000040:3.0:1761234152.653609:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234177 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234152.653613:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234152.653615:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234152.653617:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000005502b55c. 00000100:00000200:3.0:1761234152.653619:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245230848, offset 0 00000400:00000200:3.0:1761234152.653622:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234152.653628:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234152.653632:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234152.653636:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234152.653638:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234152.653640:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11824:11824:255:4294967295] 192.168.201.124@tcp LPNI seq info [11824:11824:7:4294967295] 00000400:00000200:3.0:1761234152.653645:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234152.653649:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234152.653653:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d4888078 (tot 1676778). 00000800:00000200:3.0:1761234152.653657:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234152.653659:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761234152.653661:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653662:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234152.653664:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653666:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1761234152.653668:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000100:00000040:3.0:1761234152.653669:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234152.653681:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234152.653684:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234152.653686:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234152.653687:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234152.653691:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230912:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234152.653693:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234152.653694:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234152.653694:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234152.653695:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234152.653697:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001c1b0e94. 02000000:00000001:3.0:1761234152.653698:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234152.653701:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000c9870406. 00000400:00000010:3.0:1761234152.653703:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234152.653706:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230912, portal 4 00000100:00000001:3.0:1761234152.653708:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234152.653710:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039814912 : -104717669736704 : ffffa0c2847e5f00) 00000100:00000040:3.0:1761234152.653714:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234177 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234152.653718:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234152.653724:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234152.653725:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000d31ec037. 00000100:00000200:3.0:1761234152.653727:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245230912, offset 0 00000400:00000200:3.0:1761234152.653731:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234152.653738:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234152.653742:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234152.653745:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234152.653750:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:0.0:1761234152.653751:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000200:3.0:1761234152.653761:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11825:11825:254:4294967295] 192.168.201.124@tcp LPNI seq info [11825:11825:6:4294967295] 00000400:00000200:3.0:1761234152.653778:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234152.653784:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234152.653787:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1677026). 00000400:00000200:0.0:1761234152.653789:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000800:00000200:3.0:1761234152.653796:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234152.653802:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:3.0:1761234152.653804:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653806:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234152.653809:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653812:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653815:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234152.653818:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234152.653821:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1761234152.653822:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234152.653824:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234152.653826:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234152.653828:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234153.164989:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234153.164995:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234153.164997:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234153.165002:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234153.165008:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234153.165011:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234153.165014:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234153.165016:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234153.165018:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234153.548964:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234153.548969:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234153.548978:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234153.548979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234153.548982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234153.548983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234153.549016:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234153.549023:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234153.549029:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234154.188979:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234154.188985:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234154.188986:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234154.188991:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234154.188996:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234154.189000:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234154.189003:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234154.189005:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234154.189007:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234154.573059:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234154.573065:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234154.573072:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234154.573072:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234154.573076:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234154.573077:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234154.573154:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234154.573159:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234154.573162:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234155.213031:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234155.213038:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234155.213041:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234155.213047:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234155.213054:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234155.213058:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234155.213062:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234155.213065:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234155.213068:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234155.596944:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234155.596950:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234155.596957:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234155.596958:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234155.596961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234155.596962:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234155.597057:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234155.597061:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234155.597077:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1761234155.725114:0:88983:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:0.0:1761234155.725155:0:88983:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1761234155.725162:0:88983:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:0.0:1761234155.725171:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761234155.725173:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000d8a943a (tot 1676778). 00000400:00000200:0.0:1761234155.725179:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761234155.725182:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761234155.725186:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000cfe59242 00000400:00000010:0.0:1761234155.725188:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000cfe59242. 00000100:00000001:0.0:1761234155.725192:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761234155.725197:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234177 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234155.725219:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761234155.725222:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234177 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234155.725226:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234155.725228:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761234155.725235:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761234155.725237:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d4888078 (tot 1676530). 00000400:00000200:0.0:1761234155.725238:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761234155.725240:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761234155.725242:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000005502b55c 00000400:00000010:0.0:1761234155.725243:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000005502b55c. 00000100:00000001:0.0:1761234155.725244:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761234155.725246:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234177 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234155.725250:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761234155.725251:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234177 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234155.725255:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234155.725255:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761234155.725258:0:88983:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761234155.725259:0:88983:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:0.0:1761234155.725260:0:88983:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761234155.725262:0:88983:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761234155.725263:0:88983:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000d31ec037 00000400:00000010:0.0:1761234155.725264:0:88983:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000d31ec037. 00000100:00000001:0.0:1761234155.725265:0:88983:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761234155.725267:0:88983:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234177 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234155.725270:0:88983:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761234155.725271:0:88983:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234177 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234155.725275:0:88983:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234155.725276:0:88983:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234155.725305:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234155.725308:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234155.725311:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234155.725316:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234152/real 1761234155] req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234155.725326:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234155.725327:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234155.725329:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000009aaa4f6b. 00000400:00000200:3.0:1761234155.725332:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000a390468f 00000400:00000010:3.0:1761234155.725333:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000a390468f. 00000100:00000001:3.0:1761234155.725335:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234155.725337:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234155.725342:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234155.725346:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234155.725347:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725349:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234155.725351:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234155.725353:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234155.725358:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234155.725360:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234155.725364:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234155.725366:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725367:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234155.725368:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725370:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234155.725371:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234155.725374:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234155.725377:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234155.725379:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234155.725382:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234155.725389:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230848:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234155.725393:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234155.725396:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234152/real 1761234155] req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234155.725403:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234155.725404:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234155.725405:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000c9870406. 00000400:00000200:3.0:1761234155.725407:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234155.725408:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234155.725409:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234155.725411:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234155.725422:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234155.725428:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234155.725429:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725431:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234155.725432:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234155.725434:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234155.725440:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234155.725443:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234155.725448:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234155.725449:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725451:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234155.725451:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725453:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234155.725454:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234155.725456:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234155.725459:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234155.725461:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234155.725463:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234155.725470:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245230912:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234155.725473:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234155.725476:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234152/real 1761234155] req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234155.725483:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234155.725484:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234155.725486:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234155.725488:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234155.725490:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234155.725492:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234155.725494:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234155.725500:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234155.725505:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234155.725506:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725507:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234155.725508:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234155.725511:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234155.725517:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234155.725520:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234155.725525:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234155.725527:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725528:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234155.725529:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725531:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234155.725532:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234155.725533:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234155.725536:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234155.725538:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234155.725540:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234155.725546:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002b682c08 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230784:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234155.725550:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725552:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234155.725554:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000bb0f086 x1846784245230848/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234155.725560:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234155.725563:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234155.725567:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074003a94. 02000000:00000001:3.0:1761234155.725571:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234155.725574:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234155.725576:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234155.725580:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234155.725582:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a3ed3058. 02000000:00000001:3.0:1761234155.725587:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234155.725589:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234155.725590:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000bb0f086. 00000100:00000001:3.0:1761234155.725594:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234155.725595:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725597:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234155.725599:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000632be758 x1846784245230912/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234155.725604:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234155.725605:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234155.725606:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001c1b0e94. 02000000:00000001:3.0:1761234155.725610:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234155.725611:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234155.725612:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234155.725614:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234155.725615:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:3.0:1761234155.725617:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234155.725618:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234155.725619:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000632be758. 00000100:00000001:3.0:1761234155.725622:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234155.725623:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725624:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234155.725627:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002b682c08 x1846784245230784/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234177 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234155.725632:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234155.725634:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234155.725635:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234155.725637:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234155.725638:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234155.725639:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234155.725641:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234155.725642:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:3.0:1761234155.725644:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234155.725645:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234155.725646:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002b682c08. 00000100:00000001:3.0:1761234155.725648:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234155.725649:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725651:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234155.725659:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234155.725662:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234155.725664:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234155.725666:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234156.237017:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234156.237022:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234156.237024:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234156.237028:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234156.237034:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234156.237038:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234156.237040:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234156.237043:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234156.237045:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234156.620990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234156.620998:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234156.621006:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234156.621007:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234156.621011:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234156.621013:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234156.621131:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234156.621136:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234156.621143:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234157.261021:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234157.261029:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234157.261061:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234157.261067:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234157.261074:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234157.261079:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234157.261082:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234157.261085:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234157.261087:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234157.644993:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234157.644999:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234157.645006:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234157.645009:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234157.645011:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234157.645012:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234157.645095:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234157.645098:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234157.645102:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234157.772999:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234157.773011:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234157.773013:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234157.773016:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234157.773018:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234157.773038:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234157.773045:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6626 00000100:00080000:1.0:1761234157.773049:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234157.773052:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773055:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234157.773057:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773059:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234157.773074:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234157.773076:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234157.773079:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234157.773081:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234157.773084:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234157.773086:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234157.773089:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773091:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234157.773093:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773095:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234157.773097:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234157.773100:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773103:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234157.773106:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234157.773108:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234157.773111:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773113:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234157.773116:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234157.773121:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761234157.773125:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234157.773129:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234157.773130:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773133:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234157.773136:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234157.773140:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234157.773147:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761234157.773153:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234157.773160:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234157.773170:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234157.773172:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234157.773176:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234157.773199:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773202:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234157.773204:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234157.773207:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234157.773209:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234157.773210:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234157.773211:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234157.773214:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234157.773227:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6626 00000100:00080000:1.0:1761234157.773229:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234157.773232:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773233:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234157.773235:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773237:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234157.773239:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234157.773241:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234157.773242:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234157.773244:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234157.773246:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234157.773248:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234157.773251:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773253:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234157.773256:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773259:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234157.773261:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00000001:3.0:1761234157.773263:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1761234157.773264:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773268:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.773269:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1761234157.773271:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1761234157.773273:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234157.773277:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1761234157.773281:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:1.0:1761234157.773282:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773285:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234157.773289:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234157.773294:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.773296:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234157.773297:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.773300:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234157.773303:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 02000000:00000001:3.0:1761234157.773306:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1761234157.773308:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761234157.773310:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234157.773312:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234157.773314:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234157.773317:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230976:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761234157.773318:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234157.773320:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234157.773322:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.773324:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1761234157.773328:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 02000000:00000001:3.0:1761234157.773329:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234157.773333:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773334:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234157.773339:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1761234157.773342:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:3.0:1761234157.773346:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234157.773349:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773355:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234157.773356:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234157.773357:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00000040:1.0:1761234157.773360:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:3.0:1761234157.773367:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000001:1.0:1761234157.773369:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773373:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234157.773374:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:3.0:1761234157.773375:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245230976, portal 25 00000100:00000001:1.0:1761234157.773380:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00000001:3.0:1761234157.773381:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1761234157.773382:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234157.773384:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039807616 : -104717669744000 : ffffa0c2847e4280) 00000100:00000001:1.0:1761234157.773385:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234157.773387:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234157.773391:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000040:3.0:1761234157.773395:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234182 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00080000:1.0:1761234157.773396:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6626 00000100:00080000:1.0:1761234157.773400:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234157.773405:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773409:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234157.773412:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234157.773413:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:1.0:1761234157.773415:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234157.773421:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000100:00000040:3.0:1761234157.773421:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000020:00000040:1.0:1761234157.773423:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234157.773426:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000400:00000010:3.0:1761234157.773426:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000020:00000040:1.0:1761234157.773429:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000200:3.0:1761234157.773430:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245230976, offset 0 00000020:00000001:1.0:1761234157.773433:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234157.773437:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234157.773441:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773443:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234157.773444:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773447:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000200:3.0:1761234157.773449:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000020:00000040:1.0:1761234157.773451:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234157.773455:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773458:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234157.773460:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234157.773462:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234157.773465:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773467:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000400:00000200:3.0:1761234157.773469:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000008:00100000:1.0:1761234157.773470:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234157.773474:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773476:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234157.773480:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000400:00000200:3.0:1761234157.773481:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000020:00000040:1.0:1761234157.773484:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234157.773488:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000400:00000200:3.0:1761234157.773489:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1761234157.773490:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773493:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1761234157.773494:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 02000000:00000001:1.0:1761234157.773495:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234157.773499:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234157.773502:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11826:11826:256:4294967295] 192.168.201.124@tcp LPNI seq info [11826:11826:8:4294967295] 02000000:00000010:1.0:1761234157.773513:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761234157.773518:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234157.773518:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00100000:1.0:1761234157.773526:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.0:1761234157.773532:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773539:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234157.773541:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1761234157.773543:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234157.773552:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234157.773555:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761234157.773564:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000040:1.0:1761234157.773566:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.1:1761234157.773569:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1761234157.773580:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234157.773581:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234157.773585:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000010:3.0:1761234157.773585:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000100:00000040:1.0:1761234157.773591:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6636) 00000400:00000200:3.0:1761234157.773592:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234157.773601:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234157.773616:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234157.773620:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234157.773626:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234157.773634:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234182 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.773642:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234157.773646:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234182 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.773652:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.773654:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234157.773661:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.773664:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234157.773667:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.773669:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234157.773672:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234157/real 1761234157] req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234157.773681:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234157.773683:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234157.773685:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234157.773687:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234157.773688:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234157.773691:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234157.773693:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234157.773700:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.773707:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234157.773708:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.773710:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234157.773712:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234157.773715:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.773721:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234157.773726:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234157.773733:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234157.773736:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.773737:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234157.773738:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.773741:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234157.773742:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234157.773745:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234157.773748:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234157.773750:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234157.773754:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234157.773762:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245230976:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234157.773767:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.773769:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234157.773772:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a014c997 x1846784245230976/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234157.773778:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234157.773781:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234157.773783:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234157.773787:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234157.773789:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234157.773791:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234157.773793:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234157.773795:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006d8d01a9. 02000000:00000001:3.0:1761234157.773799:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234157.773800:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234157.773801:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a014c997. 00000100:00000001:3.0:1761234157.773811:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234157.773812:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.773814:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.773823:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234157.773825:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.773827:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234157.773828:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234157.773830:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234157.773832:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.773875:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234157.773876:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234157.773878:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234157.773880:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234157.773883:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231040:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234157.773886:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234157.773888:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234157.773889:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234157.773891:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234157.773894:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234157.773895:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234157.773898:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234157.773901:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234157.773903:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231040, portal 10 00000100:00000001:3.0:1761234157.773905:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234157.773906:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039812480 : -104717669739136 : ffffa0c2847e5580) 00000100:00000040:3.0:1761234157.773912:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234182 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.773918:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234157.773921:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234157.773922:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234157.773924:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245231040, offset 0 00000400:00000200:3.0:1761234157.773928:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234157.773933:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234157.773937:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234157.773939:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234157.773942:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234157.773944:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11827:11827:256:4294967295] 192.168.201.124@tcp LPNI seq info [11827:11827:8:4294967295] 00000400:00000200:3.0:1761234157.773952:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234157.773956:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234157.773959:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234157.773964:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234157.773966:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761234157.773971:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234157.773973:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1761234157.773977:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234157.773978:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234157.773981:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234157.773984:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234157.773987:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234157.773988:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234157.773991:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234157.773993:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234182 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774000:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234157.774002:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234182 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774008:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.774010:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234157.774014:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.774015:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234157.774018:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.774020:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234157.774041:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234157/real 1761234157] req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234157.774048:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234157.774049:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234157.774050:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234157.774053:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234157.774054:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234157.774056:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234157.774058:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234157.774064:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774081:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234157.774181:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774183:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234157.774185:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234157.774189:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774196:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234157.774199:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234157.774205:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234157.774207:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774208:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234157.774209:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774211:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234157.774212:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234157.774215:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234157.774218:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234157.774220:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234157.774223:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234157.774229:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231040:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234157.774233:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234157.774236:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774241:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234157.774243:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234157.774245:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234157.774246:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234157.774249:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231104:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234157.774252:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234157.774254:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234157.774256:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234157.774258:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234157.774261:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001c1b0e94. 02000000:00000001:3.0:1761234157.774262:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234157.774265:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234157.774269:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234157.774271:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231104, portal 4 00000100:00000001:3.0:1761234157.774273:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234157.774274:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039808832 : -104717669742784 : ffffa0c2847e4740) 00000100:00000040:3.0:1761234157.774279:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234182 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774286:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234157.774288:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234157.774290:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234157.774292:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245231104, offset 0 00000400:00000200:3.0:1761234157.774296:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234157.774302:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234157.774305:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234157.774308:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234157.774310:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234157.774312:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11828:11828:256:4294967295] 192.168.201.124@tcp LPNI seq info [11828:11828:8:4294967295] 00000400:00000200:3.0:1761234157.774320:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234157.774324:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234157.774327:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234157.774331:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234157.774333:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1761234157.774338:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234157.774340:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1761234157.774344:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234157.774345:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234157.774348:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234157.774352:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234157.774355:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234157.774356:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234157.774358:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234157.774361:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234182 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774368:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234157.774370:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234182 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774376:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.774378:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234157.774381:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.774382:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234157.774385:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.774386:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234157.774390:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234157/real 1761234157] req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234157.774398:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234157.774399:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234157.774400:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234157.774402:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234157.774404:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234157.774405:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234157.774407:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234157.774414:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774421:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234157.774422:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774423:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234157.774424:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234157.774427:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234157.774439:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234157.774442:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234157.774448:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234157.774451:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774452:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234157.774453:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774455:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234157.774456:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234157.774458:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234157.774460:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234157.774462:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234157.774465:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234157.774472:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231104:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234157.774476:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774478:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234157.774480:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004567def3 x1846784245231040/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234157.774486:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234157.774488:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234157.774489:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000074a27ae0. 02000000:00000001:3.0:1761234157.774492:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234157.774494:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234157.774495:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234157.774497:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234157.774499:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f654abd8. 02000000:00000001:3.0:1761234157.774503:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234157.774504:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234157.774506:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004567def3. 00000100:00000001:3.0:1761234157.774508:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234157.774510:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774511:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234157.774513:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f33bf85c x1846784245231104/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234182 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234157.774518:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234157.774520:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234157.774521:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001c1b0e94. 02000000:00000001:3.0:1761234157.774524:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234157.774525:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234157.774527:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234157.774528:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234157.774530:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:3.0:1761234157.774532:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234157.774533:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234157.774534:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f33bf85c. 00000100:00000001:3.0:1761234157.774536:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234157.774537:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774538:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234157.774542:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234157.774544:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234157.774546:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234157.774547:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234158.285115:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234158.285120:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234158.285122:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234158.285127:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234158.285131:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234158.285135:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234158.285137:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234158.285140:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234158.285142:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234158.668991:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234158.668997:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234158.669017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234158.669019:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234158.669026:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234158.669027:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234158.669110:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234158.669114:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234158.669117:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234159.308982:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234159.308988:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234159.308990:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234159.309037:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234159.309042:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234159.309045:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234159.309047:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234159.309049:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234159.309052:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234159.693015:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234159.693023:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234159.693033:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234159.693034:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234159.693038:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234159.693040:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234159.693102:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234159.693106:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234159.693111:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234160.332924:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234160.332928:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234160.332929:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234160.332933:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234160.332937:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234160.332939:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234160.332941:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234160.332943:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234160.332945:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717011:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234160.717021:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234160.717024:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234160.717026:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717032:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234160.717035:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717036:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717039:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717043:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234160.717046:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234160.717047:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234160.717052:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234160.717054:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234160.717055:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717056:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234160.717057:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717058:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717059:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717061:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234160.717062:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234160.717063:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234160.717065:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234160.717065:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234160.717066:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717068:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234160.717069:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717070:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234160.717071:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234160.717134:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234160.717138:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234160.717141:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234161.357004:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234161.357010:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234161.357012:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234161.357017:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234161.357021:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234161.357025:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234161.357027:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234161.357029:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234161.357032:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234161.741008:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234161.741013:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234161.741020:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234161.741021:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234161.741024:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234161.741026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234161.741099:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234161.741103:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234161.741106:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234162.380966:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234162.380973:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234162.380975:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234162.380981:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234162.380987:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234162.380990:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.380993:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234162.380995:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.380997:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234162.764975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234162.764979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234162.764985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234162.764986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234162.764989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234162.764989:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234162.765051:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234162.765054:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234162.765056:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234162.892988:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234162.892995:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234162.892996:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234162.892998:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234162.893000:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234162.893005:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234162.893011:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6631 00000100:00080000:3.0:1761234162.893013:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234162.893015:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893017:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234162.893019:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893020:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234162.893023:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234162.893024:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234162.893026:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234162.893029:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234162.893030:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234162.893032:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234162.893033:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893034:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234162.893035:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893036:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234162.893038:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234162.893041:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893042:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234162.893044:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234162.893046:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234162.893048:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893049:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234162.893051:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234162.893058:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000266eaa93. 00000020:00000040:3.0:1761234162.893060:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234162.893063:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234162.893064:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893066:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234162.893068:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234162.893071:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234162.893077:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000001c1b0e94. 00000100:00000001:3.0:1761234162.893081:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234162.893087:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234162.893093:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234162.893095:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234162.893097:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000266eaa93] to pc [ptlrpcd_rcv+-1] req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234162.893116:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893117:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234162.893118:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234162.893120:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234162.893121:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234162.893122:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234162.893123:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234162.893124:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234162.893126:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6631 00000100:00080000:3.0:1761234162.893127:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234162.893128:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893129:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234162.893130:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893131:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234162.893132:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234162.893133:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234162.893133:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234162.893134:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234162.893135:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234162.893136:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234162.893138:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893139:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234162.893140:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893141:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234162.893142:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234162.893144:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893145:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234162.893146:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234162.893150:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234162.893151:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893152:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234162.893154:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234162.893156:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893157:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234162.893160:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004b242e46. 00000020:00000040:3.0:1761234162.893161:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761234162.893162:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234162.893163:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893164:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234162.893165:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234162.893166:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234162.893168:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074a27ae0. 00000100:00000001:3.0:1761234162.893169:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234162.893172:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234162.893176:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234162.893177:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234162.893180:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004b242e46] to pc [ptlrpcd_rcv+-1] req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234162.893184:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893186:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234162.893186:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234162.893188:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00000001:2.0:1761234162.893189:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1761234162.893190:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234162.893192:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234162.893193:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234162.893195:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234162.893195:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:2.0:1761234162.893197:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1761234162.893198:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6631 00000100:00080000:3.0:1761234162.893201:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234162.893205:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:3.0:1761234162.893205:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893208:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234162.893211:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234162.893214:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234162.893217:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234162.893218:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761234162.893220:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000100:00000001:2.0:1761234162.893221:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1761234162.893222:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000001:2.0:1761234162.893223:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234162.893225:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:3.0:1761234162.893227:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:2.0:1761234162.893228:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1761234162.893230:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000001:2.0:1761234162.893231:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893233:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234162.893235:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00100000:2.0:1761234162.893238:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000266eaa93 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231168:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234162.893238:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234162.893241:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00000001:2.0:1761234162.893244:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1761234162.893246:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 02000000:00000001:2.0:1761234162.893248:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1761234162.893248:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234162.893251:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234162.893251:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234162.893254:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234162.893255:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1761234162.893257:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893259:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234162.893260:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 02000000:00000010:2.0:1761234162.893261:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000070e0ffb1. 00000008:00000001:3.0:1761234162.893262:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234162.893264:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893265:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234162.893270:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000db341c88. 00000020:00000040:3.0:1761234162.893271:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000010:2.0:1761234162.893272:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 02000000:00000001:3.0:1761234162.893273:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234162.893275:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893277:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234162.893278:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1761234162.893279:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 02000000:00000001:3.0:1761234162.893279:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234162.893282:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074003a94. 00000100:00000001:3.0:1761234162.893285:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234162.893286:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231168, portal 25 00000100:00100000:3.0:1761234162.893289:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893291:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234162.893292:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091129600 : -104717618422016 : ffffa0c2878d5f00) 00000100:00000001:3.0:1761234162.893298:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234162.893299:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234162.893300:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234187 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:3.0:1761234162.893305:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000db341c88] to pc [ptlrpcd_rcv+-1] req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893318:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:3.0:1761234162.893320:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234162.893323:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:2.0:1761234162.893324:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000040:3.0:1761234162.893327:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6642) 00000400:00000010:2.0:1761234162.893328:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234162.893331:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245231168, offset 0 00000400:00000200:2.0:1761234162.893338:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234162.893351:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234162.893358:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234162.893361:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234162.893363:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234162.893365:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11829:11829:256:4294967295] 192.168.201.124@tcp LPNI seq info [11829:11829:8:4294967295] 00000400:00000200:2.0:1761234162.893372:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234162.893377:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234162.893380:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234162.893385:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234162.893388:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761234162.893392:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234162.893394:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1761234162.893397:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234162.893399:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234162.893402:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234162.893405:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234162.893408:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234162.893410:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234162.893413:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234162.893415:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234187 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893421:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234162.893423:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234187 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893427:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893428:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234162.893432:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893433:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234162.893435:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893437:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234162.893439:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234162/real 1761234162] req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234162.893445:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234162.893446:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234162.893447:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234162.893448:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234162.893449:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234162.893450:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234162.893452:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234162.893456:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893460:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234162.893461:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893462:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234162.893463:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234162.893465:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893469:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234162.893471:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234162.893479:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234162.893481:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893482:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234162.893483:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893484:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234162.893485:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234162.893487:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234162.893488:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234162.893489:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234162.893491:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234162.893496:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000266eaa93 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231168:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234162.893498:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234162.893500:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893503:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234162.893504:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234162.893505:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234162.893506:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234162.893507:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b242e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231232:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234162.893509:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234162.893510:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234162.893510:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234162.893511:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234162.893513:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234162.893514:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234162.893516:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234162.893518:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234162.893519:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231232, portal 10 00000100:00000001:2.0:1761234162.893521:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234162.893522:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091125952 : -104717618425664 : ffffa0c2878d50c0) 00000100:00000040:2.0:1761234162.893525:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234187 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893529:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234162.893530:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234162.893531:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234162.893533:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245231232, offset 0 00000400:00000200:2.0:1761234162.893535:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234162.893538:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234162.893539:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234162.893541:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234162.893542:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234162.893544:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11830:11830:256:4294967295] 192.168.201.124@tcp LPNI seq info [11830:11830:8:4294967295] 00000400:00000200:2.0:1761234162.893548:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234162.893551:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234162.893553:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234162.893555:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234162.893556:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761234162.893558:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234162.893559:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1761234162.893562:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234162.893562:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234162.893564:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234162.893566:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234162.893567:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234162.893568:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234162.893569:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234162.893571:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234187 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893575:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234162.893576:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234187 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893583:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893584:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234162.893586:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893587:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234162.893588:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893588:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234162.893590:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234162/real 1761234162] req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234162.893594:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234162.893594:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234162.893595:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234162.893596:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234162.893596:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234162.893597:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234162.893599:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234162.893603:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893607:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234162.893607:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893608:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234162.893609:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234162.893610:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893614:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234162.893615:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234162.893618:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234162.893620:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893620:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234162.893621:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893622:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234162.893623:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234162.893625:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234162.893626:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234162.893627:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234162.893629:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234162.893633:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b242e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231232:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234162.893635:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893636:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234162.893637:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000266eaa93 x1846784245231168/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234162.893641:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234162.893643:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234162.893644:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000070e0ffb1. 02000000:00000001:2.0:1761234162.893647:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234162.893648:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234162.893649:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234162.893650:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234162.893651:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000001c1b0e94. 02000000:00000001:2.0:1761234162.893654:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234162.893655:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234162.893656:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000266eaa93. 00000100:00000001:2.0:1761234162.893658:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234162.893659:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893660:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234162.893661:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004b242e46 x1846784245231232/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234162.893665:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234162.893666:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234162.893667:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234162.893668:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234162.893669:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234162.893670:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234162.893671:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234162.893672:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074a27ae0. 02000000:00000001:2.0:1761234162.893673:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234162.893674:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234162.893674:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004b242e46. 00000100:00000001:2.0:1761234162.893675:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234162.893676:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893677:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893682:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234162.893684:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893685:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234162.893685:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234162.893686:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234162.893687:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893691:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234162.893691:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234162.893693:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234162.893694:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234162.893695:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db341c88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231296:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234162.893697:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234162.893698:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234162.893698:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234162.893699:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234162.893701:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234162.893701:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234162.893703:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234162.893704:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234162.893706:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231296, portal 4 00000100:00000001:2.0:1761234162.893707:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234162.893708:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091132032 : -104717618419584 : ffffa0c2878d6880) 00000100:00000040:2.0:1761234162.893710:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234187 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893717:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234162.893719:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234162.893720:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00000200:2.0:1761234162.893721:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245231296, offset 0 00000400:00000200:2.0:1761234162.893724:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234162.893726:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234162.893728:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234162.893729:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234162.893730:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234162.893732:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11831:11831:256:4294967295] 192.168.201.124@tcp LPNI seq info [11831:11831:8:4294967295] 00000400:00000200:2.0:1761234162.893737:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234162.893740:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234162.893741:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234162.893743:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234162.893745:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761234162.893747:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234162.893748:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1761234162.893751:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234162.893752:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676282). 00000400:00000200:2.0:1761234162.893753:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234162.893755:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234162.893756:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:2.0:1761234162.893757:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:2.0:1761234162.893758:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234162.893760:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234187 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893764:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234162.893765:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234187 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893769:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893769:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234162.893771:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893772:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234162.893773:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893774:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234162.893776:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234162/real 1761234162] req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234162.893780:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234162.893781:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234162.893781:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234162.893782:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234162.893783:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234162.893784:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234162.893785:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234162.893789:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893792:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234162.893792:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893793:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234162.893794:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234162.893795:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234162.893799:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234162.893800:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234162.893803:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234162.893805:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893805:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234162.893806:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893807:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234162.893807:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234162.893809:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234162.893811:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234162.893812:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234162.893813:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234162.893817:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db341c88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231296:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234162.893819:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893820:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234162.893821:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000db341c88 x1846784245231296/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234187 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234162.893824:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234162.893825:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234162.893826:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234162.893827:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234162.893827:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234162.893828:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234162.893829:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234162.893830:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074003a94. 02000000:00000001:2.0:1761234162.893831:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234162.893831:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234162.893832:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000db341c88. 00000100:00000001:2.0:1761234162.893833:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234162.893833:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893834:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234162.893836:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234162.893836:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234162.893837:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234162.893838:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234163.404970:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234163.404978:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234163.404979:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234163.404982:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234163.404986:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234163.404989:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234163.404991:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234163.404994:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234163.404995:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234163.789005:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234163.789012:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234163.789020:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234163.789022:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234163.789027:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234163.789028:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234163.789115:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234163.789118:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234163.789122:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234164.429010:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234164.429014:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234164.429015:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234164.429019:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234164.429022:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234164.429025:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234164.429026:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234164.429028:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234164.429030:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234164.813003:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234164.813007:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234164.813013:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234164.813014:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234164.813016:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234164.813018:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234164.813089:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234164.813092:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234164.813095:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234165.452968:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234165.452974:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234165.452976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234165.452980:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234165.452985:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234165.452988:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234165.452991:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234165.452993:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234165.452996:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234165.837006:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234165.837012:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234165.837020:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234165.837021:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234165.837026:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234165.837026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234165.837091:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234165.837094:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234165.837097:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234166.476946:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234166.476959:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234166.476963:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234166.476967:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234166.476971:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234166.476974:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234166.476975:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234166.476977:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234166.476979:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234166.860967:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234166.860972:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234166.860978:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234166.860979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234166.860982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234166.860983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234166.861115:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234166.861119:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234166.861122:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234167.500923:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234167.500932:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234167.500934:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234167.500938:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234167.500944:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234167.500947:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234167.500950:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234167.500952:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234167.500954:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234167.884975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234167.884979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234167.884984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234167.884985:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234167.884987:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234167.884988:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234167.885085:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234167.885089:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234167.885093:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1761234168.012996:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234168.013005:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234168.013006:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:2.0:1761234168.013009:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234168.013010:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234168.013014:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234168.013020:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6637 00000100:00080000:2.0:1761234168.013023:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234168.013025:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013027:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234168.013029:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013030:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234168.013033:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234168.013034:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:2.0:1761234168.013036:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234168.013039:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:2.0:1761234168.013040:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:2.0:1761234168.013042:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:2.0:1761234168.013044:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013045:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234168.013046:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013047:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234168.013049:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:2.0:1761234168.013051:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013052:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234168.013054:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234168.013057:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1761234168.013058:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013060:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234168.013061:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234168.013068:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003346e953. 00000020:00000040:2.0:1761234168.013071:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:2.0:1761234168.013074:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234168.013075:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013077:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234168.013079:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234168.013082:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234168.013088:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000003562879b. 00000100:00000001:2.0:1761234168.013093:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234168.013099:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234168.013106:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234168.013108:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234168.013110:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003346e953] to pc [ptlrpcd_rcv+-1] req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234168.013129:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013131:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234168.013132:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234168.013134:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234168.013135:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1761234168.013136:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234168.013137:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1761234168.013138:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234168.013140:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6637 00000100:00080000:2.0:1761234168.013142:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1761234168.013144:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013145:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234168.013146:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013147:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234168.013149:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:2.0:1761234168.013150:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:2.0:1761234168.013151:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234168.013153:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:2.0:1761234168.013154:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:2.0:1761234168.013156:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:2.0:1761234168.013159:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013161:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234168.013162:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013163:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:2.0:1761234168.013164:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:2.0:1761234168.013166:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013168:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234168.013170:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1761234168.013170:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1761234168.013178:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234168.013178:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1761234168.013182:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013183:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1761234168.013186:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234168.013190:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234168.013194:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234168.013196:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234168.013197:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234168.013202:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000076f65824. 00000020:00000040:2.0:1761234168.013205:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:2.0:1761234168.013211:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234168.013212:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013217:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234168.013218:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234168.013220:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234168.013225:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078016eab. 00000100:00000001:3.0:1761234168.013225:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234168.013227:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013230:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234168.013234:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:2.0:1761234168.013237:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234168.013238:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234168.013249:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003346e953 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231360:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234168.013263:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1761234168.013267:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1761234168.013269:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1761234168.013270:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234168.013274:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234168.013275:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000076f65824] to pc [ptlrpcd_rcv+-1] req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234168.013279:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1761234168.013285:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013286:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:2.0:1761234168.013288:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:3.0:1761234168.013288:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234168.013291:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013292:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:2.0:1761234168.013296:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1761234168.013298:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1761234168.013300:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1761234168.013300:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00000001:2.0:1761234168.013306:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:2.0:1761234168.013309:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6637 00000100:00080000:2.0:1761234168.013311:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000010:3.0:1761234168.013311:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000040:2.0:1761234168.013318:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013320:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234168.013322:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000200:3.0:1761234168.013323:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231360, portal 25 00000100:00000001:2.0:1761234168.013324:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:2.0:1761234168.013334:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000100:00000001:3.0:1761234168.013334:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234168.013340:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090816448 : -104717618735168 : ffffa0c2878897c0) 00000020:00000040:2.0:1761234168.013341:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:2.0:1761234168.013346:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:2.0:1761234168.013349:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:2.0:1761234168.013352:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:3.0:1761234168.013354:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234193 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000020:00000001:2.0:1761234168.013356:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234168.013368:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013374:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234168.013379:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234168.013381:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:2.0:1761234168.013384:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000040:3.0:1761234168.013388:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000020:00000040:2.0:1761234168.013390:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000010:3.0:1761234168.013392:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00080000:2.0:1761234168.013395:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000200:3.0:1761234168.013396:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245231360, offset 0 00000100:00000001:2.0:1761234168.013400:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234168.013402:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234168.013404:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1761234168.013412:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:2.0:1761234168.013413:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013416:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:2.0:1761234168.013418:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1761234168.013420:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013421:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1761234168.013423:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000345661c8. 00000020:00000040:2.0:1761234168.013426:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:2.0:1761234168.013428:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234168.013430:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234168.013431:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:2.0:1761234168.013434:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234168.013436:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234168.013438:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234168.013440:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000dd1dcb0e. 00000400:00000200:3.0:1761234168.013443:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000001:2.0:1761234168.013445:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234168.013452:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00100000:2.0:1761234168.013453:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234168.013461:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000001:2.0:1761234168.013477:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234168.013478:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234168.013481:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000345661c8] to pc [ptlrpcd_rcv+-1] req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234168.013483:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11832:11832:256:4294967295] 192.168.201.124@tcp LPNI seq info [11832:11832:8:4294967295] 00000100:00000001:2.0:1761234168.013497:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.013499:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:2.0:1761234168.013503:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6647) 00000400:00000200:3.0:1761234168.013509:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234168.013516:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234168.013521:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234168.013531:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234168.013534:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1761234168.013540:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234168.013543:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1761234168.013546:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234168.013549:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234168.013556:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234168.013560:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234168.013564:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234168.013566:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234168.013570:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234168.013573:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234193 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013581:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234168.013582:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234193 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013587:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013588:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234168.013594:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013595:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234168.013597:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013599:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234168.013601:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234168/real 1761234168] req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234168.013608:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234168.013610:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234168.013611:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234168.013612:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234168.013613:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234168.013614:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234168.013615:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234168.013620:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013624:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234168.013625:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.013627:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234168.013629:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234168.013631:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013635:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234168.013638:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234168.013646:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234168.013649:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.013650:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234168.013651:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.013653:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234168.013654:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234168.013657:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234168.013659:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234168.013660:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234168.013662:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234168.013667:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003346e953 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231360:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234168.013670:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.013671:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234168.013672:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000003346e953 x1846784245231360/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234168.013676:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234168.013678:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234168.013680:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234168.013683:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234168.013684:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234168.013685:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234168.013687:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234168.013689:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000003562879b. 02000000:00000001:3.0:1761234168.013692:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234168.013693:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234168.013695:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000003346e953. 00000100:00000001:3.0:1761234168.013697:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234168.013699:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.013700:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.013708:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234168.013710:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013712:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234168.013713:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234168.013714:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234168.013716:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013720:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234168.013721:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234168.013723:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234168.013724:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234168.013726:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000076f65824 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231424:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234168.013728:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234168.013729:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234168.013729:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234168.013730:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234168.013732:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234168.013733:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234168.013735:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234168.013737:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234168.013739:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231424, portal 10 00000100:00000001:3.0:1761234168.013740:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234168.013741:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090835904 : -104717618715712 : ffffa0c28788e3c0) 00000100:00000040:3.0:1761234168.013745:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234193 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013749:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234168.013750:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234168.013752:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234168.013753:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245231424, offset 0 00000400:00000200:3.0:1761234168.013756:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234168.013758:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234168.013760:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234168.013762:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234168.013763:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234168.013765:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11833:11833:256:4294967295] 192.168.201.124@tcp LPNI seq info [11833:11833:8:4294967295] 00000400:00000200:3.0:1761234168.013771:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234168.013774:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234168.013775:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234168.013778:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234168.013779:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1761234168.013782:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234168.013783:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1761234168.013786:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234168.013787:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234168.013788:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234168.013790:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234168.013792:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234168.013793:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234168.013794:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234168.013796:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234193 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013800:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234168.013801:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234193 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013805:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013806:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234168.013807:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013808:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234168.013809:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013810:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234168.013811:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234168/real 1761234168] req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234168.013816:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234168.013816:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234168.013817:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234168.013818:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234168.013819:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234168.013820:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234168.013821:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234168.013826:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013873:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234168.013874:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.013875:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234168.013876:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234168.013878:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013883:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234168.013884:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234168.013888:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234168.013889:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.013890:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234168.013891:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.013892:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234168.013893:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234168.013894:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234168.013895:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234168.013896:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234168.013898:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234168.013902:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000076f65824 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231424:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234168.013905:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234168.013907:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013911:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234168.013912:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234168.013913:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234168.013914:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234168.013916:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000345661c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231488:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234168.013918:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234168.013919:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234168.013919:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234168.013920:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234168.013923:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234168.013924:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234168.013926:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234168.013928:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234168.013929:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231488, portal 4 00000100:00000001:3.0:1761234168.013931:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234168.013931:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356090824960 : -104717618726656 : ffffa0c28788b900) 00000100:00000040:3.0:1761234168.013934:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234193 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013938:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234168.013940:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234168.013941:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234168.013942:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245231488, offset 0 00000400:00000200:3.0:1761234168.013945:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234168.013948:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234168.013950:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234168.013952:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234168.013953:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234168.013955:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11834:11834:256:4294967295] 192.168.201.124@tcp LPNI seq info [11834:11834:8:4294967295] 00000400:00000200:3.0:1761234168.013960:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234168.013962:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234168.013964:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234168.013966:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234168.013967:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1761234168.013970:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234168.013972:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000100:3.0:1761234168.013974:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234168.013975:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234168.013977:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234168.013979:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234168.013981:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234168.013982:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234168.013983:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234168.013984:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234193 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013989:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234168.013990:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234193 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.013994:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013995:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234168.013996:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013997:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234168.013998:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.013999:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234168.014001:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234168/real 1761234168] req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234168.014005:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234168.014006:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234168.014007:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234168.014008:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234168.014009:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234168.014009:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234168.014011:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234168.014015:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.014019:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234168.014019:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.014021:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234168.014022:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234168.014024:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234168.014028:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234168.014029:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234168.014033:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234168.014034:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.014035:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234168.014035:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.014036:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234168.014037:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234168.014038:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234168.014039:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234168.014040:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234168.014041:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234168.014045:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000345661c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231488:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234168.014047:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.014048:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234168.014049:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000076f65824 x1846784245231424/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234168.014053:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234168.014054:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234168.014055:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234168.014057:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234168.014057:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234168.014058:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234168.014059:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234168.014061:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078016eab. 02000000:00000001:3.0:1761234168.014063:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234168.014064:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234168.014064:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000076f65824. 00000100:00000001:3.0:1761234168.014067:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234168.014068:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.014069:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234168.014070:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000345661c8 x1846784245231488/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234193 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234168.014074:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234168.014075:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234168.014075:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234168.014077:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234168.014077:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234168.014078:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234168.014079:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234168.014079:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000dd1dcb0e. 02000000:00000001:3.0:1761234168.014080:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234168.014081:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234168.014081:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000345661c8. 00000100:00000001:3.0:1761234168.014085:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234168.014086:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.014087:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234168.014089:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234168.014089:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234168.014090:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234168.014091:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234168.524990:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234168.524995:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234168.524997:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234168.525002:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234168.525006:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234168.525009:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234168.525010:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234168.525013:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234168.525015:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234168.908956:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234168.908960:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234168.908964:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234168.908965:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234168.908968:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234168.908969:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234168.909033:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234168.909036:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234168.909038:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234169.548913:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234169.548918:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234169.548919:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234169.548923:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234169.548927:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234169.548930:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234169.548932:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234169.548934:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234169.548935:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932930:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234169.932935:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234169.932937:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234169.932938:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932943:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234169.932945:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932946:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932947:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932951:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234169.932952:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234169.932954:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234169.932957:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234169.932959:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234169.932960:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932961:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234169.932962:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932963:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932964:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932967:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234169.932968:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234169.932969:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234169.932972:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234169.932973:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234169.932974:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932975:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234169.932976:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932977:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234169.932978:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234169.933055:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234169.933057:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234169.933060:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234170.572975:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234170.572980:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234170.572982:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234170.572986:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234170.572990:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234170.572994:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234170.572996:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234170.572998:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234170.573001:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234170.956945:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234170.956951:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234170.956959:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234170.956961:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234170.956965:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234170.956967:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234170.957034:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234170.957037:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234170.957040:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234171.596947:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234171.596951:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234171.596952:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234171.596956:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234171.596960:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234171.596963:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234171.596965:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234171.596967:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234171.596969:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234171.980967:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234171.980973:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234171.980981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234171.980982:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234171.980985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234171.980986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234171.981085:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234171.981088:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234171.981091:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234172.620948:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234172.620954:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234172.620956:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234172.620961:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234172.620966:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234172.620969:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234172.620971:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234172.620973:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234172.620977:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234173.004923:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234173.004928:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234173.004934:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234173.004936:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234173.004939:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234173.004940:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234173.005008:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234173.005012:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234173.005015:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234173.133008:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234173.133019:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234173.133021:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234173.133023:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234173.133026:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234173.133031:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234173.133037:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6642 00000100:00080000:0.0:1761234173.133040:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234173.133042:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133044:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234173.133045:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133047:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234173.133049:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234173.133051:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234173.133053:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234173.133055:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234173.133057:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234173.133058:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234173.133060:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133061:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234173.133063:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133063:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234173.133065:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234173.133067:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133069:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234173.133071:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234173.133073:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234173.133075:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133076:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234173.133077:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234173.133085:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761234173.133095:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234173.133098:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234173.133099:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133101:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234173.133103:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234173.133106:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234173.133114:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761234173.133119:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234173.133124:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234173.133130:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234173.133132:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234173.133135:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234173.133152:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133154:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234173.133155:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234173.133157:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234173.133158:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234173.133159:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234173.133160:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234173.133162:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234173.133163:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6642 00000100:00080000:0.0:1761234173.133164:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234173.133165:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133166:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234173.133167:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133168:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234173.133169:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234173.133170:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234173.133171:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234173.133172:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234173.133173:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234173.133175:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234173.133176:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133178:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234173.133178:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133179:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234173.133181:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234173.133182:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133183:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234173.133184:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234173.133189:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234173.133190:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133191:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234173.133193:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234173.133196:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133197:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234173.133199:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761234173.133201:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234173.133203:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234173.133204:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133207:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234173.133208:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234173.133209:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234173.133211:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761234173.133213:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234173.133215:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234173.133220:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234173.133222:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133222:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234173.133231:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1761234173.133232:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133237:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1761234173.133243:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133245:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234173.133246:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1761234173.133248:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234173.133255:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234173.133256:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761234173.133260:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234173.133261:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234173.133266:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234173.133271:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6642 00000100:00000001:3.0:1761234173.133273:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1761234173.133274:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1761234173.133276:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234173.133279:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133283:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1761234173.133284:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1761234173.133285:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133288:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 02000000:00000001:3.0:1761234173.133288:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234173.133295:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234173.133297:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000100:00100000:3.0:1761234173.133298:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000efaa8dcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231552:192.168.201.124@tcp:250: 00000020:00000001:0.0:1761234173.133301:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000100:00000001:3.0:1761234173.133303:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1761234173.133305:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 02000000:00000001:3.0:1761234173.133308:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1761234173.133311:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:3.0:1761234173.133312:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234173.133319:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1761234173.133321:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761234173.133329:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000010:3.0:1761234173.133329:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234173.133332:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133337:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1761234173.133341:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00000040:0.0:1761234173.133343:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133348:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000010:3.0:1761234173.133350:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000020:00000040:0.0:1761234173.133353:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761234173.133360:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000200:3.0:1761234173.133360:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231552, portal 25 00000100:00000001:3.0:1761234173.133364:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1761234173.133367:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133367:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246318848 : -104717463232768 : ffffa0c290cd5f00) 02000000:00000001:0.0:1761234173.133377:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:3.0:1761234173.133380:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234198 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 02000000:00000001:0.0:1761234173.133382:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234173.133390:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133393:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:0.0:1761234173.133395:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000040:3.0:1761234173.133398:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000008:00100000:0.0:1761234173.133400:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1761234173.133401:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234173.133405:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245231552, offset 0 00000008:00000001:0.0:1761234173.133408:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133411:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234173.133414:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000010:0.0:1761234173.133420:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761234173.133424:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234173.133428:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234173.133429:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234173.133431:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1761234173.133433:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 02000000:00000001:0.0:1761234173.133434:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234173.133438:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234173.133442:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 02000000:00000010:0.0:1761234173.133443:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000400:00000200:3.0:1761234173.133449:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1761234173.133450:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234173.133453:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00100000:0.0:1761234173.133462:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234173.133463:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11835:11835:256:4294967295] 192.168.201.124@tcp LPNI seq info [11835:11835:8:4294967295] 00000100:00000001:0.0:1761234173.133481:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234173.133483:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234173.133487:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234173.133500:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:0.0:1761234173.133517:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1761234173.133518:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234173.133527:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000010:3.0:1761234173.133529:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000100:00000040:0.0:1761234173.133532:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6652) 00000800:00000200:3.0:1761234173.133543:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234173.133547:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1761234173.133554:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234173.133557:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1761234173.133561:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234173.133563:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234173.133570:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234173.133574:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234173.133578:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234173.133579:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234173.133583:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234173.133586:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234198 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133594:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234173.133596:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234198 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133600:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133602:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234173.133608:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133609:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234173.133612:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133614:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234173.133616:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234173/real 1761234173] req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234173.133622:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234173.133623:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234173.133624:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234173.133625:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234173.133626:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234173.133628:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234173.133630:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234173.133639:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133643:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234173.133644:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.133646:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234173.133648:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234173.133650:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133653:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234173.133656:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234173.133660:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234173.133662:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.133663:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234173.133663:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.133665:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234173.133667:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234173.133669:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234173.133671:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234173.133673:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234173.133674:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234173.133679:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000efaa8dcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231552:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234173.133683:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.133684:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234173.133685:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000efaa8dcd x1846784245231552/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234173.133689:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234173.133692:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234173.133694:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234173.133696:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234173.133698:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234173.133700:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234173.133701:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234173.133703:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000005851ba37. 02000000:00000001:3.0:1761234173.133707:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234173.133709:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234173.133710:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000efaa8dcd. 00000100:00000001:3.0:1761234173.133713:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234173.133714:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.133715:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.133724:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234173.133725:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133727:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234173.133728:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234173.133729:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234173.133746:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133751:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234173.133753:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234173.133754:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234173.133755:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234173.133757:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000080840a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231616:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234173.133759:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234173.133760:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234173.133761:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234173.133762:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234173.133764:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234173.133765:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234173.133767:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234173.133769:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234173.133771:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231616, portal 10 00000100:00000001:3.0:1761234173.133772:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234173.133773:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246309120 : -104717463242496 : ffffa0c290cd3900) 00000100:00000040:3.0:1761234173.133778:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234198 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133783:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234173.133784:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234173.133786:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234173.133787:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245231616, offset 0 00000400:00000200:3.0:1761234173.133790:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234173.133793:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234173.133795:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234173.133796:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234173.133798:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234173.133799:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11836:11836:256:4294967295] 192.168.201.124@tcp LPNI seq info [11836:11836:8:4294967295] 00000400:00000200:3.0:1761234173.133805:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234173.133808:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234173.133809:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234173.133812:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234173.133813:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1761234173.133816:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234173.133818:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1761234173.133820:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234173.133821:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234173.133823:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234173.133825:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234173.133827:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234173.133828:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234173.133829:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234173.133831:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234198 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133836:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234173.133837:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234198 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133841:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133842:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234173.133874:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133876:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234173.133877:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.133879:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234173.133881:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234173/real 1761234173] req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234173.133886:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234173.133886:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234173.133887:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234173.133888:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234173.133889:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234173.133891:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234173.133892:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234173.133896:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133900:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234173.133900:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.133901:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234173.133902:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234173.133903:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133906:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234173.133908:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234173.133912:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234173.133913:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.133914:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234173.133915:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.133916:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234173.133917:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234173.133919:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234173.133921:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234173.133922:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234173.133924:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234173.133928:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000080840a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231616:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234173.133930:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234173.133932:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133936:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234173.133936:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234173.133937:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234173.133938:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234173.133940:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000581658b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231680:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234173.133942:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234173.133943:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234173.133944:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234173.133945:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234173.133947:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234173.133948:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234173.133950:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234173.133952:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234173.133954:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231680, portal 4 00000100:00000001:3.0:1761234173.133955:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234173.133956:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246320064 : -104717463231552 : ffffa0c290cd63c0) 00000100:00000040:3.0:1761234173.133959:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234198 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.133962:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234173.133964:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234173.133965:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234173.133966:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245231680, offset 0 00000400:00000200:3.0:1761234173.133969:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234173.133972:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234173.133975:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234173.133976:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234173.133978:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234173.133979:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11837:11837:256:4294967295] 192.168.201.124@tcp LPNI seq info [11837:11837:8:4294967295] 00000400:00000200:3.0:1761234173.133985:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234173.133987:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234173.133988:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234173.133990:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234173.133992:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1761234173.133994:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234173.133996:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1761234173.133999:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234173.134000:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234173.134002:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234173.134004:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234173.134005:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234173.134006:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234173.134007:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234173.134009:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234198 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.134013:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234173.134014:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234198 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.134018:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.134019:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234173.134020:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.134021:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234173.134022:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.134024:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234173.134025:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234173/real 1761234173] req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234173.134029:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234173.134030:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234173.134031:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234173.134032:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234173.134032:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234173.134034:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234173.134035:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234173.134039:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.134042:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234173.134043:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.134044:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234173.134044:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234173.134045:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234173.134049:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234173.134050:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234173.134053:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234173.134055:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.134055:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234173.134056:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.134057:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234173.134057:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234173.134058:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234173.134060:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234173.134061:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234173.134063:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234173.134067:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000581658b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231680:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234173.134071:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.134073:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234173.134074:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000080840a x1846784245231616/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234173.134078:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234173.134079:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234173.134079:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234173.134080:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234173.134081:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234173.134082:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234173.134083:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234173.134084:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000007c64e188. 02000000:00000001:3.0:1761234173.134086:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234173.134086:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234173.134087:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000080840a. 00000100:00000001:3.0:1761234173.134089:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234173.134090:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.134091:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234173.134092:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000581658b4 x1846784245231680/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234198 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234173.134097:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234173.134097:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234173.134098:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234173.134100:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234173.134101:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234173.134101:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234173.134102:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234173.134102:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:3.0:1761234173.134104:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234173.134104:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234173.134105:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000581658b4. 00000100:00000001:3.0:1761234173.134106:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234173.134107:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.134108:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234173.134109:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234173.134111:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234173.134112:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234173.134113:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234173.644989:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234173.644994:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234173.644995:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234173.644999:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234173.645004:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234173.645007:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234173.645009:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234173.645011:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234173.645013:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234174.029007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234174.029012:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234174.029017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234174.029018:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234174.029021:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234174.029022:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1761234174.029130:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1761234174.029137:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1761234174.029143:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234174.668987:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234174.668992:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234174.668994:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234174.668997:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234174.669001:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234174.669004:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234174.669009:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234174.669010:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234174.669012:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234175.052959:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234175.052974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234175.052983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234175.052984:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234175.052989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234175.052991:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234175.053069:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234175.053072:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234175.053076:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234175.692987:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234175.692991:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234175.692992:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234175.692996:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234175.692999:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234175.693002:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234175.693004:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234175.693006:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234175.693008:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234176.076960:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234176.076966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234176.076974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234176.076975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234176.076979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234176.076980:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234176.077053:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234176.077057:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234176.077059:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234176.716960:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234176.716967:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234176.716969:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234176.716974:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234176.716979:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234176.716983:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234176.716985:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234176.716988:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234176.716990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234177.100896:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234177.100901:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234177.100908:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234177.100911:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234177.100913:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234177.100914:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234177.100961:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234177.100964:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234177.100967:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234177.741011:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234177.741017:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234177.741018:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234177.741024:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234177.741030:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234177.741033:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234177.741035:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234177.741038:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234177.741040:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234178.124961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234178.124967:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234178.124975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234178.124976:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234178.124979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234178.124980:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234178.125048:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234178.125052:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234178.125056:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234178.252945:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234178.252955:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234178.252956:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234178.252959:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234178.252960:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234178.252965:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234178.252970:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6647 00000100:00080000:0.0:1761234178.252972:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234178.252974:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.252977:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234178.252979:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.252980:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234178.252983:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234178.252985:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234178.252987:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234178.252989:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234178.252990:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234178.252992:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234178.252993:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.252995:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234178.252996:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.252997:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234178.252999:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234178.253001:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253002:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234178.253004:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234178.253006:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234178.253008:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234178.253009:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234178.253011:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234178.253019:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761234178.253022:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234178.253024:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234178.253026:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234178.253028:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234178.253029:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234178.253032:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234178.253038:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761234178.253042:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234178.253047:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234178.253058:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234178.253060:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234178.253062:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234178.253078:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234178.253079:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234178.253080:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234178.253083:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234178.253083:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234178.253084:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234178.253085:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234178.253087:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234178.253088:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6647 00000100:00080000:0.0:1761234178.253089:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234178.253091:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253092:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234178.253093:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253094:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234178.253095:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234178.253095:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234178.253096:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234178.253097:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234178.253098:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234178.253100:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234178.253101:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253102:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234178.253104:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253105:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234178.253106:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234178.253109:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253111:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234178.253112:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234178.253116:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234178.253117:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253118:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1761234178.253119:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234178.253139:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1761234178.253139:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000008:00000001:0.0:1761234178.253142:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253143:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1761234178.253144:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234178.253148:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000100:00000040:3.0:1761234178.253150:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000020:00000040:0.0:1761234178.253151:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234178.253155:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234178.253155:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234178.253158:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234178.253160:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234178.253161:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253163:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 02000000:00000010:0.0:1761234178.253164:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:3.0:1761234178.253166:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234178.253168:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234178.253170:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234178.253173:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234178.253174:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761234178.253179:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c218247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231744:192.168.201.124@tcp:250: 00000100:00000001:0.0:1761234178.253185:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234178.253186:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253190:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1761234178.253191:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234178.253194:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234178.253196:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234178.253198:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234178.253199:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1761234178.253201:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234178.253203:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:3.0:1761234178.253204:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 00000100:00000001:0.0:1761234178.253205:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234178.253207:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 02000000:00000001:3.0:1761234178.253208:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234178.253209:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234178.253211:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234178.253213:0:468:0:(import.c:519:import_select_connection()) Process entered 00000400:00000010:3.0:1761234178.253214:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00080000:0.0:1761234178.253216:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6647 00000100:00080000:0.0:1761234178.253219:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234178.253220:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253222:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1761234178.253222:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000040:0.0:1761234178.253225:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253227:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234178.253229:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234178.253230:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000100:00000200:3.0:1761234178.253230:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231744, portal 25 00000020:00000001:0.0:1761234178.253232:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234178.253234:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000001:3.0:1761234178.253234:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1761234178.253236:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:3.0:1761234178.253237:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246306688 : -104717463244928 : ffffa0c290cd2f80) 00000020:00000001:0.0:1761234178.253239:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761234178.253242:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253244:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234178.253245:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253246:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234178.253247:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761234178.253249:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000040:3.0:1761234178.253251:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234203 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234178.253252:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234178.253256:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234178.253258:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234178.253261:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253263:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:0.0:1761234178.253264:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234178.253266:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234178.253270:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234178.253271:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234178.253272:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234178.253274:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000010:0.0:1761234178.253277:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000100:00000200:3.0:1761234178.253278:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245231744, offset 0 00000020:00000040:0.0:1761234178.253280:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234178.253283:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234178.253283:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234178.253285:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234178.253286:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234178.253287:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234178.253288:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000010:0.0:1761234178.253293:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761234178.253298:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234178.253302:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00100000:0.0:1761234178.253305:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234178.253312:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000001:0.0:1761234178.253317:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1761234178.253317:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1761234178.253319:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234178.253321:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000040:0.0:1761234178.253327:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234178.253327:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11838:11838:256:4294967295] 192.168.201.124@tcp LPNI seq info [11838:11838:8:4294967295] 00000100:00000001:0.0:1761234178.253339:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234178.253343:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000200:3.0:1761234178.253343:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000040:0.0:1761234178.253348:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6657) 00000800:00000200:3.0:1761234178.253349:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234178.253355:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234178.253360:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234178.253363:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1761234178.253368:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234178.253371:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1761234178.253374:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234178.253376:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234178.253380:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234178.253384:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234178.253387:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234178.253388:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234178.253392:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234178.253394:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234203 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253399:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234178.253401:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234203 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253405:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253406:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234178.253411:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253412:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234178.253414:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253416:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234178.253418:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234178/real 1761234178] req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234178.253424:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234178.253425:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234178.253426:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234178.253427:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234178.253428:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234178.253429:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234178.253431:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234178.253436:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253439:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234178.253441:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253442:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234178.253443:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234178.253445:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253449:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234178.253452:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234178.253456:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234178.253458:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253459:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234178.253460:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253461:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234178.253462:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234178.253464:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234178.253466:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234178.253467:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234178.253469:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234178.253474:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c218247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231744:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234178.253477:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253478:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234178.253479:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000003c218247 x1846784245231744/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234178.253484:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234178.253485:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234178.253487:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234178.253490:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234178.253491:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234178.253492:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234178.253493:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234178.253495:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078138607. 02000000:00000001:3.0:1761234178.253497:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234178.253498:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234178.253499:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000003c218247. 00000100:00000001:3.0:1761234178.253501:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234178.253502:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253503:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253511:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234178.253513:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253515:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234178.253516:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234178.253517:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234178.253520:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253527:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234178.253529:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234178.253530:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234178.253532:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234178.253535:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070caacfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231808:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234178.253538:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234178.253539:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234178.253540:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234178.253542:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234178.253545:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234178.253547:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234178.253550:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234178.253553:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234178.253571:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231808, portal 10 00000100:00000001:3.0:1761234178.253573:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234178.253574:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246301824 : -104717463249792 : ffffa0c290cd1c80) 00000100:00000040:3.0:1761234178.253579:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234203 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253586:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234178.253589:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234178.253591:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234178.253593:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245231808, offset 0 00000400:00000200:3.0:1761234178.253597:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234178.253602:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234178.253605:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234178.253607:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234178.253620:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234178.253623:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11839:11839:256:4294967295] 192.168.201.124@tcp LPNI seq info [11839:11839:8:4294967295] 00000400:00000200:3.0:1761234178.253630:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234178.253661:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234178.253663:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234178.253666:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234178.253667:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1761234178.253670:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234178.253671:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1761234178.253674:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234178.253675:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234178.253677:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234178.253679:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234178.253680:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234178.253681:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234178.253683:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234178.253685:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234203 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253689:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234178.253691:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234203 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253695:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253696:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234178.253698:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253699:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234178.253700:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253701:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234178.253703:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234178/real 1761234178] req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234178.253708:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234178.253708:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234178.253709:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234178.253710:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234178.253711:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234178.253712:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234178.253713:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234178.253718:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253721:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234178.253722:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253723:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234178.253723:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234178.253725:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253728:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234178.253730:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234178.253733:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234178.253734:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253735:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234178.253736:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253737:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234178.253737:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234178.253739:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234178.253740:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234178.253741:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234178.253743:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234178.253747:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070caacfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231808:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234178.253749:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234178.253750:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253757:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234178.253758:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234178.253759:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234178.253761:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234178.253762:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eff3cf06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231872:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234178.253763:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234178.253764:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234178.253765:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234178.253766:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234178.253768:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234178.253769:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234178.253770:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234178.253772:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234178.253773:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231872, portal 4 00000100:00000001:3.0:1761234178.253774:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234178.253775:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246294528 : -104717463257088 : ffffa0c290cd0000) 00000100:00000040:3.0:1761234178.253778:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234203 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253782:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234178.253783:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234178.253784:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234178.253785:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245231872, offset 0 00000400:00000200:3.0:1761234178.253788:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234178.253791:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234178.253793:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234178.253794:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234178.253795:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234178.253797:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11840:11840:256:4294967295] 192.168.201.124@tcp LPNI seq info [11840:11840:8:4294967295] 00000400:00000200:3.0:1761234178.253802:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234178.253804:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234178.253806:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234178.253808:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234178.253810:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1761234178.253812:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234178.253813:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1761234178.253816:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234178.253817:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234178.253818:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234178.253820:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234178.253821:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234178.253822:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234178.253823:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234178.253824:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234203 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253829:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234178.253830:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234203 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253833:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253834:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234178.253835:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253836:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234178.253837:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253838:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234178.253839:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234178/real 1761234178] req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234178.253861:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234178.253862:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234178.253863:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234178.253864:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234178.253865:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234178.253866:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234178.253868:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234178.253888:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253896:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234178.253897:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253898:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234178.253899:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234178.253901:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234178.253904:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234178.253906:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234178.253910:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234178.253911:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253912:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234178.253912:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253914:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234178.253914:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234178.253916:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234178.253918:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234178.253919:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234178.253921:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234178.253925:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eff3cf06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245231872:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234178.253927:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253928:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234178.253929:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000070caacfc x1846784245231808/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234178.253933:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234178.253934:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234178.253935:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234178.253937:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234178.253938:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234178.253939:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234178.253941:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234178.253942:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000040aa0f54. 02000000:00000001:3.0:1761234178.253943:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234178.253944:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234178.253945:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000070caacfc. 00000100:00000001:3.0:1761234178.253947:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234178.253947:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253949:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234178.253950:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000eff3cf06 x1846784245231872/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234203 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234178.253953:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234178.253954:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234178.253955:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234178.253956:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234178.253956:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234178.253957:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234178.253958:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234178.253959:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:3.0:1761234178.253960:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234178.253960:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234178.253961:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000eff3cf06. 00000100:00000001:3.0:1761234178.253962:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234178.253963:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253964:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234178.253966:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234178.253967:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234178.253968:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234178.253969:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234178.764983:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234178.765001:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234178.765003:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234178.765008:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234178.765013:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234178.765017:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234178.765019:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234178.765021:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234178.765024:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234179.148985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234179.148992:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234179.149001:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234179.149004:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234179.149008:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234179.149009:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234179.149105:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234179.149109:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234179.149112:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234179.789005:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234179.789012:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234179.789014:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234179.789020:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234179.789028:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234179.789034:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234179.789037:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234179.789041:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234179.789044:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.172958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234180.172967:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234180.172970:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234180.172973:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.172978:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234180.172981:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.172983:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.172986:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.172990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234180.172992:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234180.172993:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234180.172998:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234180.173001:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234180.173002:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.173004:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234180.173005:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.173006:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.173008:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.173012:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234180.173013:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234180.173014:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234180.173016:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234180.173017:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234180.173018:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.173019:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234180.173020:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.173022:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234180.173023:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234180.173119:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234180.173122:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234180.173125:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234180.812962:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234180.812968:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234180.812970:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234180.812975:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234180.812979:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234180.812983:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234180.812986:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234180.812989:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234180.812992:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234181.197028:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234181.197035:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234181.197044:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234181.197046:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234181.197049:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234181.197050:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234181.197148:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234181.197153:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234181.197156:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234181.837019:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234181.837027:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234181.837029:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234181.837034:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234181.837041:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234181.837045:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234181.837048:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234181.837051:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234181.837054:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234182.220975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234182.220981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234182.220988:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234182.220989:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234182.220992:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234182.220994:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234182.221053:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234182.221056:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234182.221060:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234182.860937:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234182.860942:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234182.860943:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234182.860946:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234182.860950:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234182.860953:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234182.860954:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234182.860956:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234182.860957:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234183.244970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234183.244974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234183.244981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234183.244982:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234183.244985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234183.244986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234183.245075:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234183.245077:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234183.245080:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234183.372972:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234183.372985:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234183.372987:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234183.372990:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234183.372993:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234183.372999:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234183.373006:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6652 00000100:00080000:0.0:1761234183.373009:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234183.373012:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373014:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234183.373016:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373017:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234183.373020:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234183.373022:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234183.373024:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234183.373028:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234183.373030:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234183.373032:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234183.373034:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373037:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234183.373038:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373040:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234183.373042:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234183.373045:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373046:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234183.373049:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234183.373052:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234183.373054:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373056:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234183.373057:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234183.373068:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000006edc786f. 00000020:00000040:0.0:1761234183.373071:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234183.373075:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234183.373076:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373079:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234183.373082:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234183.373086:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234183.373094:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761234183.373101:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234183.373108:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234183.373116:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234183.373118:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234183.373121:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000006edc786f] to pc [ptlrpcd_rcv+-1] req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234183.373142:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373144:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234183.373145:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234183.373148:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234183.373148:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234183.373150:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234183.373151:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234183.373153:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234183.373155:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6652 00000100:00080000:0.0:1761234183.373157:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234183.373158:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373159:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234183.373160:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373161:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234183.373162:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234183.373163:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234183.373164:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234183.373165:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234183.373166:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234183.373167:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234183.373169:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373170:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234183.373170:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373171:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234183.373172:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234183.373174:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373176:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234183.373177:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234183.373183:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234183.373184:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373186:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234183.373188:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234183.373191:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373192:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234183.373194:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ad5ae63. 00000020:00000040:0.0:1761234183.373196:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234183.373197:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234183.373199:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373200:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234183.373201:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234183.373202:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234183.373204:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761234183.373206:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234183.373208:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234183.373213:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234183.373215:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234183.373217:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ad5ae63] to pc [ptlrpcd_rcv+-1] req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234183.373222:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373224:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234183.373226:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234183.373229:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234183.373230:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761234183.373230:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234183.373231:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234183.373233:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234183.373237:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6652 00000100:00080000:0.0:1761234183.373239:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234183.373240:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373241:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234183.373243:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373246:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:3.0:1761234183.373247:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1761234183.373253:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234183.373254:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234183.373255:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000100:00000001:3.0:1761234183.373256:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1761234183.373257:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761234183.373259:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:3.0:1761234183.373260:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1761234183.373262:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761234183.373269:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373271:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234183.373276:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000040:3.0:1761234183.373278:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234183.373280:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234183.373287:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761234183.373295:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234183.373301:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234183.373305:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234183.373308:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234183.373312:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373316:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234183.373318:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234183.373325:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373329:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373329:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234183.373334:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234183.373338:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 02000000:00000001:3.0:1761234183.373339:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1761234183.373343:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:3.0:1761234183.373344:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234183.373345:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234183.373346:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373349:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234183.373351:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1761234183.373351:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000006edc786f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231936:192.168.201.124@tcp:250: 02000000:00000001:0.0:1761234183.373355:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234183.373365:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:3.0:1761234183.373367:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1761234183.373372:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234183.373373:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234183.373377:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234183.373380:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234183.373382:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1761234183.373387:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234183.373388:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234183.373390:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:3.0:1761234183.373391:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234183.373398:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234183.373410:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00000001:0.0:1761234183.373412:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234183.373420:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000010:3.0:1761234183.373425:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000040:0.0:1761234183.373426:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6662) 00000100:00000200:3.0:1761234183.373436:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245231936, portal 25 00000100:00000001:3.0:1761234183.373441:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234183.373442:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246305472 : -104717463246144 : ffffa0c290cd2ac0) 00000100:00000040:3.0:1761234183.373448:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234208 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373454:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234183.373457:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234183.373459:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234183.373461:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245231936, offset 0 00000400:00000200:3.0:1761234183.373466:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234183.373477:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234183.373483:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234183.373486:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234183.373488:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234183.373490:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11841:11841:256:4294967295] 192.168.201.124@tcp LPNI seq info [11841:11841:8:4294967295] 00000400:00000200:3.0:1761234183.373498:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234183.373502:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234183.373505:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234183.373511:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234183.373513:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761234183.373518:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234183.373520:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1761234183.373523:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234183.373525:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234183.373529:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234183.373532:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234183.373535:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234183.373550:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234183.373554:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234183.373556:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234208 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373563:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234183.373565:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234208 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373569:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373570:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234183.373575:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373576:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234183.373578:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373580:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234183.373582:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234183/real 1761234183] req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234183.373588:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234183.373589:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234183.373590:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234183.373591:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234183.373592:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234183.373593:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234183.373595:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234183.373599:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373603:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234183.373604:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373606:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234183.373607:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234183.373609:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373613:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234183.373615:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234183.373619:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234183.373620:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373621:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234183.373622:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373624:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234183.373625:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234183.373627:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234183.373630:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234183.373631:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234183.373633:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234183.373637:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000006edc786f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245231936:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234183.373640:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234183.373642:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373646:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234183.373647:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234183.373649:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234183.373650:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234183.373651:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ad5ae63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232000:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234183.373653:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234183.373654:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234183.373656:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234183.373657:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234183.373658:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234183.373659:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234183.373661:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234183.373663:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234183.373664:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232000, portal 10 00000100:00000001:3.0:1761234183.373665:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234183.373666:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246316416 : -104717463235200 : ffffa0c290cd5580) 00000100:00000040:3.0:1761234183.373670:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234208 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373674:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234183.373675:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234183.373676:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234183.373678:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245232000, offset 0 00000400:00000200:3.0:1761234183.373680:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234183.373682:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234183.373685:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234183.373686:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234183.373687:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234183.373688:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11842:11842:256:4294967295] 192.168.201.124@tcp LPNI seq info [11842:11842:8:4294967295] 00000400:00000200:3.0:1761234183.373693:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234183.373696:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234183.373698:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234183.373700:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234183.373701:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761234183.373704:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234183.373705:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1761234183.373707:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234183.373709:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234183.373710:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234183.373712:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234183.373714:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234183.373715:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234183.373716:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234183.373718:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234208 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373722:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234183.373723:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234208 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373726:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373727:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234183.373728:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373729:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234183.373730:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373731:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234183.373732:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234183/real 1761234183] req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234183.373736:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234183.373737:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234183.373738:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234183.373739:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234183.373739:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234183.373741:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234183.373742:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234183.373746:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373749:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234183.373749:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373750:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234183.373751:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234183.373752:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373756:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234183.373757:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234183.373761:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234183.373763:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373764:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234183.373764:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373765:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234183.373766:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234183.373767:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234183.373769:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234183.373770:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234183.373771:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234183.373775:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ad5ae63 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232000:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234183.373777:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373778:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234183.373779:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000006edc786f x1846784245231936/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234183.373783:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234183.373785:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234183.373786:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234183.373788:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234183.373789:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234183.373791:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234183.373792:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234183.373793:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000007c64e188. 02000000:00000001:3.0:1761234183.373796:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234183.373797:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234183.373798:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000006edc786f. 00000100:00000001:3.0:1761234183.373801:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234183.373801:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373802:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234183.373804:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000009ad5ae63 x1846784245232000/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234183.373808:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234183.373808:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234183.373809:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234183.373811:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234183.373812:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234183.373812:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234183.373813:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234183.373814:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000005851ba37. 02000000:00000001:3.0:1761234183.373815:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234183.373816:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234183.373816:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000009ad5ae63. 00000100:00000001:3.0:1761234183.373818:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234183.373818:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373819:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373825:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234183.373826:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373828:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234183.373829:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234183.373829:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234183.373831:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373835:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234183.373835:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234183.373837:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234183.373837:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234183.373839:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eff3cf06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232064:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234183.373840:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234183.373841:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234183.373870:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234183.373872:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234183.373874:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234183.373875:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234183.373877:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234183.373878:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234183.373880:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232064, portal 4 00000100:00000001:3.0:1761234183.373881:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234183.373882:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246294528 : -104717463257088 : ffffa0c290cd0000) 00000100:00000040:3.0:1761234183.373885:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234208 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373889:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234183.373891:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234183.373892:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234183.373893:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245232064, offset 0 00000400:00000200:3.0:1761234183.373895:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234183.373898:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234183.373900:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234183.373902:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234183.373903:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234183.373905:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11843:11843:256:4294967295] 192.168.201.124@tcp LPNI seq info [11843:11843:8:4294967295] 00000400:00000200:3.0:1761234183.373909:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234183.373912:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234183.373914:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234183.373916:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234183.373917:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1761234183.373919:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234183.373921:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1761234183.373923:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234183.373924:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234183.373926:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234183.373927:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234183.373929:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234183.373930:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234183.373931:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234183.373932:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234208 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373936:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234183.373938:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234208 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373942:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373943:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234183.373944:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373945:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234183.373946:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.373947:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234183.373948:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234183/real 1761234183] req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234183.373953:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234183.373954:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234183.373955:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234183.373956:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234183.373956:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234183.373958:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234183.373959:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234183.373963:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373966:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234183.373967:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373968:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234183.373968:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234183.373970:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234183.373973:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234183.373974:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234183.373978:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234183.373979:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373980:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234183.373981:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373982:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234183.373983:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234183.373985:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234183.373986:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234183.373987:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234183.373989:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234183.373993:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eff3cf06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232064:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234183.373994:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.373995:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234183.373996:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000eff3cf06 x1846784245232064/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234208 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234183.374000:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234183.374000:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234183.374001:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234183.374003:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234183.374003:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234183.374004:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234183.374005:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234183.374006:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:3.0:1761234183.374007:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234183.374008:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234183.374009:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000eff3cf06. 00000100:00000001:3.0:1761234183.374010:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234183.374011:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.374012:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234183.374013:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234183.374014:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234183.374015:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234183.374016:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234183.884968:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234183.884974:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234183.884975:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234183.884980:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234183.884984:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234183.884988:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234183.884990:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234183.884993:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234183.884995:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234184.269052:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234184.269056:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234184.269061:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234184.269062:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234184.269064:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234184.269065:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234184.269148:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234184.269151:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234184.269155:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234184.908924:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234184.908928:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234184.908929:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234184.908933:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234184.908937:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234184.908940:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234184.908942:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234184.908943:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234184.908945:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234185.292980:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234185.292984:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234185.292990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234185.292991:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234185.292995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234185.292996:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234185.293097:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234185.293100:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234185.293103:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234185.932961:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234185.932965:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234185.932966:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234185.932970:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234185.932974:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234185.932977:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234185.932979:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234185.932981:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234185.932982:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234186.316975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234186.316981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234186.316988:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234186.316990:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234186.316993:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234186.316994:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234186.317094:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234186.317096:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234186.317099:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234186.956981:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234186.956986:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234186.956989:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234186.956993:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234186.956999:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234186.957003:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234186.957006:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234186.957008:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234186.957010:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234187.340981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234187.340987:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234187.340995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234187.340996:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234187.340999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234187.341000:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234187.341076:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234187.341081:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234187.341085:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234187.980939:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234187.980945:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234187.980947:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234187.980953:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234187.980958:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234187.980961:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234187.980963:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234187.980965:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234187.980968:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234188.364953:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234188.364958:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234188.364962:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234188.364963:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234188.364965:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234188.364966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234188.365024:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234188.365027:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234188.365029:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234188.492970:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234188.492981:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234188.492983:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234188.492985:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234188.492987:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234188.492992:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234188.492998:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6657 00000100:00080000:0.0:1761234188.493001:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234188.493004:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493008:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234188.493010:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493013:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234188.493016:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234188.493018:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234188.493021:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234188.493024:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234188.493026:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234188.493028:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234188.493031:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493033:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234188.493035:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493036:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234188.493039:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234188.493042:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493044:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234188.493047:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234188.493050:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234188.493052:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493054:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234188.493056:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234188.493065:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000020:00000040:0.0:1761234188.493068:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234188.493072:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234188.493074:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493077:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234188.493079:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234188.493082:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234188.493090:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761234188.493095:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234188.493102:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234188.493119:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234188.493121:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234188.493124:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234188.493146:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493148:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234188.493149:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234188.493153:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234188.493154:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234188.493155:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234188.493156:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234188.493159:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234188.493161:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6657 00000100:00080000:0.0:1761234188.493162:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234188.493164:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493166:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234188.493168:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493169:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234188.493171:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234188.493172:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234188.493174:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234188.493175:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234188.493177:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234188.493179:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234188.493181:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493182:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234188.493184:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493185:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234188.493187:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234188.493190:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493192:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234188.493194:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234188.493199:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234188.493201:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493203:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234188.493206:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234188.493209:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493211:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234188.493214:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761234188.493217:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234188.493220:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234188.493221:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493221:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1761234188.493225:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234188.493226:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1761234188.493227:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1761234188.493229:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493230:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000010:0.0:1761234188.493234:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000040:3.0:1761234188.493237:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234188.493239:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234188.493247:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493255:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234188.493257:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493259:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234188.493261:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234188.493262:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1761234188.493265:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234188.493266:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234188.493274:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070caacfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232128:192.168.201.124@tcp:250: 00000100:00000001:0.0:1761234188.493280:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493281:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1761234188.493283:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234188.493285:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1761234188.493287:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1761234188.493289:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234188.493291:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 02000000:00000001:3.0:1761234188.493291:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493294:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1761234188.493295:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1761234188.493296:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234188.493300:0:468:0:(import.c:519:import_select_connection()) Process entered 02000000:00000010:3.0:1761234188.493302:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 00000100:00080000:0.0:1761234188.493304:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6657 02000000:00000001:3.0:1761234188.493306:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1761234188.493308:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234188.493312:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493314:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1761234188.493314:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00000040:0.0:1761234188.493316:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493319:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234188.493322:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234188.493323:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000400:00000010:3.0:1761234188.493323:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000020:00000001:0.0:1761234188.493325:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234188.493328:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761234188.493330:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761234188.493332:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000200:3.0:1761234188.493332:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232128, portal 25 00000100:00000001:3.0:1761234188.493337:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000040:0.0:1761234188.493339:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234188.493339:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246301824 : -104717463249792 : ffffa0c290cd1c80) 00000100:00000001:0.0:1761234188.493344:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234188.493346:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234213 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:0.0:1761234188.493348:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493352:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234188.493356:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00000001:3.0:1761234188.493358:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00080000:0.0:1761234188.493361:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000040:3.0:1761234188.493363:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493365:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234188.493366:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 02000000:00000001:0.0:1761234188.493370:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:3.0:1761234188.493370:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245232128, offset 0 02000000:00000001:0.0:1761234188.493373:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1761234188.493378:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:0.0:1761234188.493379:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493382:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234188.493384:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234188.493386:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493388:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234188.493391:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000010:0.0:1761234188.493392:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761234188.493396:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234188.493399:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000400:00000200:3.0:1761234188.493399:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 02000000:00000001:0.0:1761234188.493402:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234188.493405:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1761234188.493405:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1761234188.493408:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1761234188.493408:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 02000000:00000001:0.0:1761234188.493411:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234188.493413:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11844:11844:256:4294967295] 192.168.201.124@tcp LPNI seq info [11844:11844:8:4294967295] 02000000:00000010:0.0:1761234188.493417:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761234188.493423:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234188.493423:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00100000:0.0:1761234188.493430:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.0:1761234188.493431:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234188.493437:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000100:00000001:0.0:1761234188.493455:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234188.493457:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1761234188.493460:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000040:0.0:1761234188.493463:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000200:3.0:1761234188.493464:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761234188.493473:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234188.493476:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000100:00000001:0.0:1761234188.493480:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1761234188.493483:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234188.493484:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000010:3.0:1761234188.493486:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000100:00000040:0.0:1761234188.493489:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6667) 00000400:00000200:3.0:1761234188.493492:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234188.493497:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234188.493500:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234188.493501:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234188.493505:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234188.493508:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234213 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493515:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234188.493517:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234213 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493521:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493522:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234188.493527:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493528:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234188.493530:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493532:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234188.493533:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234188/real 1761234188] req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234188.493539:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234188.493540:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234188.493542:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234188.493543:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234188.493544:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234188.493545:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234188.493547:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234188.493551:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493554:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234188.493555:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493557:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234188.493558:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234188.493560:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493563:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234188.493565:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234188.493569:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234188.493571:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493572:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234188.493573:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493574:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234188.493575:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234188.493577:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234188.493579:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234188.493580:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234188.493582:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234188.493586:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070caacfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232128:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234188.493589:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493590:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234188.493592:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000070caacfc x1846784245232128/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234188.493596:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234188.493597:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234188.493599:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234188.493602:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234188.493603:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234188.493604:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234188.493605:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234188.493607:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000040aa0f54. 02000000:00000001:3.0:1761234188.493610:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234188.493611:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234188.493612:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000070caacfc. 00000100:00000001:3.0:1761234188.493614:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234188.493614:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493616:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493621:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234188.493622:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493624:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234188.493625:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234188.493626:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234188.493627:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493631:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234188.493632:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234188.493633:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234188.493634:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234188.493636:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c218247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232192:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234188.493637:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234188.493638:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234188.493639:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234188.493640:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234188.493642:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234188.493643:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234188.493645:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234188.493647:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234188.493649:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232192, portal 10 00000100:00000001:3.0:1761234188.493650:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234188.493651:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246306688 : -104717463244928 : ffffa0c290cd2f80) 00000100:00000040:3.0:1761234188.493654:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234213 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493658:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234188.493660:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234188.493661:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234188.493663:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245232192, offset 0 00000400:00000200:3.0:1761234188.493665:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234188.493668:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234188.493671:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234188.493672:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234188.493673:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234188.493675:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11845:11845:256:4294967295] 192.168.201.124@tcp LPNI seq info [11845:11845:8:4294967295] 00000400:00000200:3.0:1761234188.493680:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234188.493683:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234188.493684:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234188.493687:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234188.493688:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761234188.493691:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234188.493692:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1761234188.493695:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234188.493696:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234188.493697:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234188.493699:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234188.493701:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234188.493702:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234188.493703:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234188.493704:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234213 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493708:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234188.493710:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234213 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493713:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493714:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234188.493716:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493717:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234188.493718:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493719:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234188.493720:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234188/real 1761234188] req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234188.493724:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234188.493725:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234188.493725:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234188.493726:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234188.493727:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234188.493728:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234188.493729:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234188.493733:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493737:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234188.493737:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493738:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234188.493739:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234188.493740:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493744:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234188.493745:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234188.493748:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234188.493750:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493750:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234188.493751:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493752:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234188.493753:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234188.493755:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234188.493756:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234188.493757:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234188.493759:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234188.493763:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c218247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232192:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234188.493764:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234188.493766:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493769:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234188.493770:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234188.493771:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234188.493771:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234188.493773:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000581658b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232256:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234188.493774:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234188.493775:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234188.493776:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234188.493777:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234188.493778:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234188.493779:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234188.493781:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234188.493783:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234188.493784:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232256, portal 4 00000100:00000001:3.0:1761234188.493785:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234188.493786:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246320064 : -104717463231552 : ffffa0c290cd63c0) 00000100:00000040:3.0:1761234188.493788:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234213 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493792:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234188.493794:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234188.493795:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234188.493796:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245232256, offset 0 00000400:00000200:3.0:1761234188.493798:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234188.493801:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234188.493803:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234188.493805:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234188.493806:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234188.493807:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11846:11846:256:4294967295] 192.168.201.124@tcp LPNI seq info [11846:11846:8:4294967295] 00000400:00000200:3.0:1761234188.493812:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234188.493814:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234188.493816:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234188.493818:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234188.493819:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1761234188.493821:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234188.493822:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1761234188.493824:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234188.493826:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234188.493827:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234188.493829:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234188.493830:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234188.493831:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234188.493832:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234188.493833:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234213 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493837:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234188.493838:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234213 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493863:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493864:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234188.493866:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493867:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234188.493868:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493870:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234188.493872:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234188/real 1761234188] req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234188.493876:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234188.493877:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234188.493878:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234188.493879:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234188.493880:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234188.493881:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234188.493882:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234188.493887:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493890:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234188.493890:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493891:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234188.493892:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234188.493893:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234188.493897:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234188.493898:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234188.493902:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234188.493903:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493904:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234188.493904:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493905:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234188.493906:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234188.493907:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234188.493909:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234188.493910:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234188.493912:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234188.493916:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000581658b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232256:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234188.493918:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493919:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234188.493920:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000003c218247 x1846784245232192/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234188.493924:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234188.493925:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234188.493925:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234188.493927:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234188.493927:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234188.493928:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234188.493929:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234188.493930:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078138607. 02000000:00000001:3.0:1761234188.493931:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234188.493932:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234188.493932:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000003c218247. 00000100:00000001:3.0:1761234188.493934:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234188.493935:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493936:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234188.493937:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000581658b4 x1846784245232256/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234213 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234188.493940:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234188.493941:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234188.493941:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234188.493942:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234188.493943:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234188.493944:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234188.493944:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234188.493945:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:3.0:1761234188.493946:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234188.493947:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234188.493947:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000581658b4. 00000100:00000001:3.0:1761234188.493949:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234188.493950:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493951:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234188.493953:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234188.493953:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234188.493954:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234188.493955:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234189.004920:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234189.004924:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234189.004926:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234189.004929:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234189.004933:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234189.004936:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234189.004937:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234189.004939:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234189.004941:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234189.388941:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234189.388946:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234189.388952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234189.388954:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234189.388958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234189.388959:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234189.389031:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234189.389035:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234189.389038:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234190.028968:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234190.028973:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234190.028975:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234190.028978:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234190.028982:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234190.028985:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234190.028987:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234190.028988:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234190.028990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412931:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234190.412936:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234190.412938:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234190.412940:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412944:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234190.412946:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412948:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412949:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234190.412953:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234190.412954:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234190.412958:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234190.412959:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234190.412960:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412961:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234190.412962:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412963:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412964:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234190.412967:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234190.412967:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234190.412969:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234190.412969:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234190.412971:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412972:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234190.412972:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412973:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234190.412975:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234190.413022:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234190.413025:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234190.413027:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234191.052982:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234191.052988:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234191.052990:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234191.052994:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234191.052998:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234191.053002:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234191.053005:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234191.053007:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234191.053009:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234191.437012:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234191.437018:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234191.437026:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234191.437027:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234191.437031:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234191.437033:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234191.437104:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234191.437108:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234191.437111:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234192.076975:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234192.076981:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234192.076982:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234192.076986:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234192.076991:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234192.076995:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234192.076997:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234192.076999:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234192.077001:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234192.460914:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234192.460918:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234192.460923:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234192.460924:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234192.460927:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234192.460928:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234192.461034:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234192.461041:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234192.461046:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234193.101012:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234193.101017:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234193.101019:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234193.101028:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234193.101033:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234193.101036:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234193.101038:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234193.101040:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234193.101042:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234193.484967:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234193.484972:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234193.484977:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234193.484978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234193.484981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234193.484981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234193.485059:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234193.485063:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234193.485066:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234193.612964:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234193.612977:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234193.612979:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234193.612982:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234193.612985:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234193.612990:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234193.612996:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6662 00000100:00080000:1.0:1761234193.613000:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234193.613003:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613006:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234193.613008:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613011:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234193.613015:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234193.613017:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234193.613020:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234193.613022:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234193.613024:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234193.613026:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234193.613027:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613029:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234193.613029:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613031:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234193.613032:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234193.613035:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613037:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234193.613039:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234193.613042:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234193.613044:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613045:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234193.613047:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234193.613055:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761234193.613058:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234193.613061:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234193.613062:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613065:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234193.613067:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234193.613070:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234193.613077:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234193.613082:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234193.613088:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234193.613095:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234193.613097:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234193.613099:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234193.613122:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613125:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234193.613127:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234193.613131:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234193.613132:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234193.613133:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234193.613135:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234193.613137:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234193.613140:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6662 00000100:00080000:1.0:1761234193.613142:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234193.613145:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613147:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234193.613148:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613150:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234193.613152:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234193.613154:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234193.613156:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234193.613158:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234193.613160:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234193.613162:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234193.613166:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613168:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234193.613170:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613172:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234193.613174:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00000001:3.0:1761234193.613175:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1761234193.613177:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613182:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613182:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1761234193.613185:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1761234193.613186:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234193.613191:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1761234193.613195:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:1.0:1761234193.613196:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613199:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234193.613203:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234193.613209:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613209:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234193.613210:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613212:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234193.613215:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 02000000:00000001:3.0:1761234193.613218:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1761234193.613219:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234193.613222:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234193.613223:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234193.613224:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613226:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234193.613228:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234193.613230:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234193.613231:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232320:192.168.201.124@tcp:250: 02000000:00000010:1.0:1761234193.613233:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761234193.613238:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613240:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1761234193.613244:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234193.613246:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234193.613248:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613250:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234193.613252:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234193.613253:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1761234193.613255:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:3.0:1761234193.613260:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 00000100:00000001:1.0:1761234193.613261:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613265:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 02000000:00000001:3.0:1761234193.613265:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761234193.613267:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234193.613271:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234193.613272:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234193.613273:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234193.613273:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234193.613275:0:468:0:(import.c:519:import_select_connection()) Process entered 00000400:00000010:3.0:1761234193.613276:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00080000:1.0:1761234193.613277:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6662 00000100:00080000:1.0:1761234193.613281:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234193.613283:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613284:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234193.613285:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613286:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000010:3.0:1761234193.613286:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000020:00000001:1.0:1761234193.613291:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234193.613292:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234193.613293:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234193.613295:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234193.613296:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234193.613297:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000200:3.0:1761234193.613297:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232320, portal 25 00000100:00000040:1.0:1761234193.613303:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234193.613303:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234193.613306:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039795456 : -104717669756160 : ffffa0c2847e1300) 00000100:00000001:1.0:1761234193.613307:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234193.613309:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613312:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234193.613314:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234193.613318:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000040:3.0:1761234193.613318:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234218 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234193.613319:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234193.613322:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234193.613324:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234193.613327:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613330:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234193.613331:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234193.613334:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613337:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613337:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000010:1.0:1761234193.613341:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761234193.613358:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234193.613361:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000040:3.0:1761234193.613362:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 02000000:00000001:1.0:1761234193.613363:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234193.613366:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234193.613367:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234193.613368:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234193.613369:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 02000000:00000010:1.0:1761234193.613371:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761234193.613375:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1761234193.613375:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245232320, offset 0 00000100:00100000:1.0:1761234193.613381:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234193.613392:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234193.613396:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234193.613398:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234193.613402:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234193.613409:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234193.613411:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:1.0:1761234193.613412:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234193.613418:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6672) 00000400:00000200:3.0:1761234193.613422:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234193.613427:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234193.613431:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234193.613436:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11847:11847:256:4294967295] 192.168.201.124@tcp LPNI seq info [11847:11847:8:4294967295] 00000400:00000200:3.0:1761234193.613448:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234193.613454:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234193.613460:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234193.613466:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234193.613470:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761234193.613476:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234193.613479:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1761234193.613484:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234193.613486:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234193.613492:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234193.613498:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234193.613502:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234193.613504:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234193.613508:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234193.613511:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234218 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613526:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234193.613528:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234218 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613536:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613537:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234193.613544:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613545:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234193.613548:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613551:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234193.613554:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234193/real 1761234193] req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234193.613563:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234193.613565:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234193.613567:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234193.613569:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234193.613570:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234193.613572:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234193.613575:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234193.613583:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613589:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234193.613591:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.613593:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234193.613595:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234193.613598:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613604:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234193.613608:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234193.613614:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234193.613617:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.613619:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234193.613620:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.613622:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234193.613624:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234193.613627:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234193.613630:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234193.613632:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234193.613635:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234193.613643:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232320:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234193.613648:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.613650:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234193.613652:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fd54b549 x1846784245232320/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234193.613659:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234193.613661:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234193.613663:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234193.613666:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234193.613668:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234193.613669:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234193.613672:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234193.613674:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:3.0:1761234193.613677:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234193.613678:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234193.613679:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fd54b549. 00000100:00000001:3.0:1761234193.613682:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234193.613683:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.613685:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.613694:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234193.613696:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613698:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234193.613699:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234193.613700:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234193.613703:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613709:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234193.613711:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234193.613713:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234193.613715:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234193.613718:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000643267a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232384:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234193.613721:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234193.613723:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234193.613724:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234193.613726:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234193.613728:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234193.613730:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234193.613733:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234193.613736:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234193.613739:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232384, portal 10 00000100:00000001:3.0:1761234193.613741:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234193.613742:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039800320 : -104717669751296 : ffffa0c2847e2600) 00000100:00000040:3.0:1761234193.613747:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234218 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613753:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234193.613756:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234193.613758:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234193.613760:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245232384, offset 0 00000400:00000200:3.0:1761234193.613764:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234193.613784:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234193.613788:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234193.613791:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234193.613793:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234193.613795:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11848:11848:256:4294967295] 192.168.201.124@tcp LPNI seq info [11848:11848:8:4294967295] 00000400:00000200:3.0:1761234193.613803:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234193.613808:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234193.613811:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234193.613816:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234193.613818:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761234193.613823:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234193.613825:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1761234193.613829:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234193.613830:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234193.613833:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234193.613837:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234193.613840:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234193.613841:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234193.613872:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234193.613876:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234218 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613883:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234193.613885:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234218 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613892:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613893:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234193.613897:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613899:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234193.613901:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.613903:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234193.613905:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234193/real 1761234193] req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234193.613913:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234193.613914:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234193.613916:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234193.613918:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234193.613919:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234193.613921:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234193.613923:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234193.613930:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613942:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234193.613943:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.613945:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234193.613946:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234193.613949:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.613955:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234193.613958:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234193.613964:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234193.613966:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.613968:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234193.613969:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.613970:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234193.613972:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234193.613974:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234193.613976:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234193.613979:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234193.613982:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234193.613989:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000643267a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232384:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234193.613993:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234193.613995:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.614001:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234193.614002:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234193.614005:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234193.614006:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234193.614009:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d6a5077 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232448:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234193.614012:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234193.614014:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234193.614015:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234193.614016:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234193.614020:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234193.614021:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234193.614024:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234193.614027:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234193.614030:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232448, portal 4 00000100:00000001:3.0:1761234193.614032:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234193.614033:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039794240 : -104717669757376 : ffffa0c2847e0e40) 00000100:00000040:3.0:1761234193.614038:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234218 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.614044:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234193.614047:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234193.614049:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234193.614051:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245232448, offset 0 00000400:00000200:3.0:1761234193.614055:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234193.614060:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234193.614064:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234193.614066:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234193.614069:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234193.614071:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11849:11849:256:4294967295] 192.168.201.124@tcp LPNI seq info [11849:11849:8:4294967295] 00000400:00000200:3.0:1761234193.614079:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234193.614083:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234193.614086:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234193.614090:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234193.614092:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1761234193.614096:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234193.614099:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1761234193.614102:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234193.614104:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234193.614107:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234193.614110:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234193.614113:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234193.614115:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234193.614117:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234193.614119:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234218 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.614126:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234193.614129:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234218 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.614135:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.614136:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234193.614139:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.614141:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234193.614143:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.614144:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234193.614146:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234193/real 1761234193] req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234193.614154:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234193.614155:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234193.614156:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234193.614158:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234193.614159:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234193.614161:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234193.614163:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234193.614170:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.614176:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234193.614177:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.614178:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234193.614180:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234193.614182:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234193.614195:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234193.614198:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234193.614204:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234193.614206:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.614208:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234193.614209:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.614210:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234193.614211:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234193.614213:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234193.614216:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234193.614218:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234193.614221:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234193.614228:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d6a5077 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232448:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234193.614232:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.614234:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234193.614236:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000643267a x1846784245232384/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234193.614243:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234193.614244:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234193.614245:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234193.614248:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234193.614249:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234193.614250:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234193.614252:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234193.614253:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:3.0:1761234193.614256:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234193.614257:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234193.614259:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000643267a. 00000100:00000001:3.0:1761234193.614261:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234193.614262:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.614264:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234193.614266:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007d6a5077 x1846784245232448/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234218 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234193.614272:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234193.614273:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234193.614274:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234193.614277:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234193.614278:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234193.614279:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234193.614280:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234193.614281:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f654abd8. 02000000:00000001:3.0:1761234193.614284:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234193.614285:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234193.614286:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007d6a5077. 00000100:00000001:3.0:1761234193.614288:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234193.614289:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.614291:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234193.614294:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234193.614295:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234193.614297:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234193.614298:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234194.124961:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234194.124969:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234194.124971:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234194.124978:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234194.124984:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234194.124988:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234194.124991:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234194.124994:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234194.124997:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234194.508924:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234194.508928:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234194.508934:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234194.508935:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234194.508937:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234194.508938:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234194.509036:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234194.509040:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234194.509044:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234195.149012:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234195.149019:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234195.149020:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234195.149026:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234195.149031:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234195.149035:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234195.149037:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234195.149041:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234195.149043:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234195.532991:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234195.532997:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234195.533004:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234195.533005:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234195.533007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234195.533008:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234195.533118:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234195.533121:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234195.533123:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234196.172935:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234196.172939:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234196.172941:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234196.172945:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234196.172949:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234196.172952:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234196.172953:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234196.172955:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234196.172957:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234196.556943:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234196.556948:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234196.556952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234196.556954:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234196.556956:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234196.556957:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234196.557025:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234196.557028:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234196.557031:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234197.196977:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234197.196983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234197.196985:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234197.196989:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234197.196995:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234197.196998:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234197.197001:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234197.197003:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234197.197005:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234197.580964:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234197.580969:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234197.580974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234197.580974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234197.580977:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234197.580977:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234197.581092:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234197.581097:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234197.581100:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234198.220961:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234198.220967:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234198.220969:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234198.220974:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234198.220979:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234198.220983:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234198.220985:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234198.220987:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234198.220990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234198.604934:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234198.604938:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234198.604943:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234198.604944:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234198.604947:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234198.604948:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234198.605043:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234198.605049:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234198.605055:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234198.732952:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234198.732963:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234198.732965:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234198.732968:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234198.732970:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234198.732975:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234198.732982:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6667 00000100:00080000:1.0:1761234198.732986:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234198.732989:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234198.732992:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234198.732994:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234198.732996:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234198.733000:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234198.733002:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234198.733004:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234198.733007:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234198.733009:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234198.733011:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234198.733013:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234198.733014:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234198.733015:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234198.733016:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234198.733017:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234198.733019:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234198.733021:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234198.733024:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234198.733027:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234198.733029:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733030:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234198.733032:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234198.733040:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761234198.733043:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234198.733046:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234198.733047:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733049:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234198.733052:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234198.733055:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234198.733061:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761234198.733066:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234198.733071:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234198.733078:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234198.733080:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234198.733083:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234198.733099:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733101:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234198.733102:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234198.733104:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234198.733105:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234198.733106:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234198.733107:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234198.733108:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234198.733110:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6667 00000100:00080000:1.0:1761234198.733111:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234198.733112:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234198.733113:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234198.733114:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234198.733116:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234198.733117:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234198.733118:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234198.733119:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234198.733120:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234198.733121:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234198.733122:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234198.733124:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234198.733124:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234198.733125:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234198.733126:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234198.733127:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234198.733129:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234198.733130:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234198.733131:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234198.733135:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234198.733136:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733138:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234198.733140:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234198.733142:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733143:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234198.733145:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761234198.733146:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234198.733148:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234198.733148:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733149:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234198.733150:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234198.733151:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234198.733153:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761234198.733155:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234198.733159:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234198.733167:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234198.733168:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234198.733170:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234198.733174:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733175:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234198.733176:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234198.733178:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234198.733178:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234198.733179:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234198.733180:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234198.733181:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:3.0:1761234198.733182:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1761234198.733183:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6667 00000100:00080000:1.0:1761234198.733187:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234198.733188:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234198.733189:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761234198.733190:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234198.733193:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234198.733193:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1761234198.733197:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234198.733200:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234198.733201:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234198.733203:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234198.733204:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000040:3.0:1761234198.733206:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000020:00000001:1.0:1761234198.733208:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234198.733210:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234198.733221:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234198.733228:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234198.733233:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234198.733235:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234198.733239:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:3.0:1761234198.733239:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1761234198.733262:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 02000000:00000001:3.0:1761234198.733263:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234198.733269:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761234198.733270:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00100000:3.0:1761234198.733275:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005940a194 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232512:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761234198.733277:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733281:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761234198.733284:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234198.733287:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1761234198.733289:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234198.733290:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234198.733294:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1761234198.733298:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733300:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234198.733301:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234198.733304:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234198.733304:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234198.733309:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733311:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234198.733318:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000400:00000010:3.0:1761234198.733318:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000020:00000040:1.0:1761234198.733324:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000010:3.0:1761234198.733327:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 02000000:00000001:1.0:1761234198.733329:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234198.733332:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733334:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234198.733336:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:3.0:1761234198.733336:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232512, portal 25 02000000:00000001:1.0:1761234198.733340:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733341:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234198.733346:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039820992 : -104717669730624 : ffffa0c2847e76c0) 02000000:00000010:1.0:1761234198.733350:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000040:3.0:1761234198.733353:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234223 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234198.733357:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733361:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234198.733365:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234198.733368:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00100000:1.0:1761234198.733369:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234198.733372:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245232512, offset 0 00000100:00000001:1.0:1761234198.733382:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234198.733384:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234198.733386:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000040:1.0:1761234198.733390:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234198.733396:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234198.733398:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234198.733403:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6677) 00000400:00000200:3.0:1761234198.733406:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234198.733414:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234198.733418:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234198.733422:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234198.733427:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11850:11850:256:4294967295] 192.168.201.124@tcp LPNI seq info [11850:11850:8:4294967295] 00000400:00000200:3.0:1761234198.733437:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234198.733443:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234198.733447:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234198.733454:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234198.733457:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761234198.733463:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234198.733465:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1761234198.733469:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234198.733471:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234198.733478:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234198.733482:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234198.733486:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234198.733488:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234198.733492:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234198.733495:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234223 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733503:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234198.733505:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234223 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733509:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733510:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234198.733517:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733518:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234198.733521:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733522:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234198.733524:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234198/real 1761234198] req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234198.733531:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234198.733533:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234198.733534:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234198.733535:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234198.733536:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234198.733538:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234198.733540:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234198.733545:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733548:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234198.733549:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733551:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234198.733552:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234198.733555:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733558:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234198.733561:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234198.733565:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234198.733567:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733568:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234198.733568:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733570:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234198.733572:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234198.733574:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234198.733577:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234198.733579:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234198.733583:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234198.733602:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005940a194 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232512:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234198.733611:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234198.733614:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733621:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234198.733623:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234198.733625:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234198.733627:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234198.733629:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3b47c4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232576:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234198.733633:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234198.733634:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234198.733635:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234198.733637:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234198.733639:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234198.733640:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234198.733643:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234198.733645:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234198.733647:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232576, portal 10 00000100:00000001:3.0:1761234198.733648:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234198.733649:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039818560 : -104717669733056 : ffffa0c2847e6d40) 00000100:00000040:3.0:1761234198.733653:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234223 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733658:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234198.733660:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234198.733661:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234198.733662:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245232576, offset 0 00000400:00000200:3.0:1761234198.733665:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234198.733668:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234198.733671:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234198.733672:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234198.733674:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234198.733676:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11851:11851:256:4294967295] 192.168.201.124@tcp LPNI seq info [11851:11851:8:4294967295] 00000400:00000200:3.0:1761234198.733682:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234198.733685:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234198.733687:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234198.733689:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234198.733691:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761234198.733694:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234198.733695:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1761234198.733698:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234198.733699:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234198.733701:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234198.733703:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234198.733705:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234198.733706:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234198.733707:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234198.733708:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234223 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733713:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234198.733714:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234223 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733719:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733719:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234198.733721:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733722:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234198.733723:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733724:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234198.733726:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234198/real 1761234198] req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234198.733731:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234198.733732:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234198.733733:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234198.733734:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234198.733734:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234198.733735:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234198.733736:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234198.733740:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733744:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234198.733745:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733747:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234198.733748:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234198.733749:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733754:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234198.733755:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234198.733759:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234198.733760:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733761:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234198.733761:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733762:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234198.733763:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234198.733764:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234198.733766:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234198.733767:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234198.733768:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234198.733772:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3b47c4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232576:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234198.733775:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733776:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234198.733777:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000005940a194 x1846784245232512/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234198.733782:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234198.733785:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234198.733786:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234198.733790:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234198.733791:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234198.733793:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234198.733795:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234198.733797:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:3.0:1761234198.733801:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234198.733802:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234198.733803:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000005940a194. 00000100:00000001:3.0:1761234198.733806:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234198.733807:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733808:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234198.733809:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000c3b47c4a x1846784245232576/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234198.733813:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234198.733814:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234198.733815:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234198.733816:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234198.733816:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234198.733817:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234198.733818:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234198.733818:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:3.0:1761234198.733820:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234198.733820:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234198.733821:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000c3b47c4a. 00000100:00000001:3.0:1761234198.733822:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234198.733823:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733824:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.733832:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234198.733833:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.733835:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234198.733836:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234198.733837:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234198.733838:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733899:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234198.733901:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234198.733903:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234198.733904:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234198.733908:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f4f489ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232640:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234198.733911:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234198.733914:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234198.733915:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234198.733917:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234198.733919:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234198.733922:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234198.733925:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234198.733927:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234198.733930:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232640, portal 4 00000100:00000001:3.0:1761234198.733932:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234198.733933:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039793024 : -104717669758592 : ffffa0c2847e0980) 00000100:00000040:3.0:1761234198.733938:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234223 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.733944:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234198.733947:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234198.733949:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234198.733951:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245232640, offset 0 00000400:00000200:3.0:1761234198.733956:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234198.733961:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234198.733965:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234198.733967:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234198.733969:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234198.733972:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11852:11852:256:4294967295] 192.168.201.124@tcp LPNI seq info [11852:11852:8:4294967295] 00000400:00000200:3.0:1761234198.733980:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234198.733985:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234198.733988:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234198.733992:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234198.733997:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1761234198.734001:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234198.734004:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1761234198.734007:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234198.734009:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234198.734012:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234198.734015:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234198.734019:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234198.734020:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234198.734022:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234198.734024:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234223 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.734031:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234198.734033:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234223 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.734039:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.734041:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234198.734043:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.734045:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234198.734047:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.734049:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234198.734051:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234198/real 1761234198] req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234198.734058:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234198.734059:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234198.734060:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234198.734061:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234198.734063:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234198.734065:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234198.734066:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234198.734074:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.734079:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234198.734080:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.734082:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234198.734083:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234198.734085:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234198.734090:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234198.734092:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234198.734096:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234198.734098:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.734099:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234198.734099:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.734100:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234198.734101:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234198.734102:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234198.734104:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234198.734105:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234198.734107:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234198.734111:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f4f489ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232640:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234198.734113:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.734114:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234198.734115:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f4f489ba x1846784245232640/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234223 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234198.734119:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234198.734119:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234198.734120:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234198.734122:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234198.734122:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234198.734123:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234198.734124:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234198.734124:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a3ed3058. 02000000:00000001:3.0:1761234198.734127:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234198.734128:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234198.734128:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f4f489ba. 00000100:00000001:3.0:1761234198.734130:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234198.734131:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.734132:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234198.734135:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234198.734136:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234198.734137:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234198.734138:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234199.244961:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234199.244966:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234199.244968:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234199.244973:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234199.244978:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234199.244981:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234199.244984:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234199.244986:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234199.244988:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234199.628965:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234199.628969:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234199.628974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234199.628975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234199.628977:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234199.628978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234199.629076:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234199.629079:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234199.629082:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234200.268935:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234200.268940:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234200.268941:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234200.268944:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234200.268948:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234200.268951:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234200.268953:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234200.268954:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234200.268956:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.652966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234200.652973:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234200.652975:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234200.652978:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.652983:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234200.652985:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.652987:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.652990:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.652995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234200.652996:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234200.652998:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234200.653003:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234200.653005:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234200.653007:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.653008:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234200.653009:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.653011:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.653013:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.653017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234200.653019:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234200.653020:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234200.653022:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234200.653023:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234200.653024:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.653026:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234200.653027:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.653028:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234200.653030:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234200.653101:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234200.653105:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234200.653108:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234201.292969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234201.292974:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234201.292976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234201.292980:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234201.292984:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234201.292988:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234201.292990:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234201.292991:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234201.292993:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234201.676966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234201.676970:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234201.676975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234201.676976:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234201.676979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234201.676979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234201.677040:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234201.677043:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234201.677045:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234202.316978:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234202.316983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234202.316985:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234202.316989:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234202.316993:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234202.316996:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234202.316998:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234202.317001:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234202.317003:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234202.700963:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234202.700968:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234202.700976:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234202.700978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234202.700982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234202.700984:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234202.701084:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234202.701088:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234202.701090:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234203.340959:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234203.340964:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234203.340965:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234203.340969:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234203.340972:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234203.340975:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234203.340977:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234203.340979:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234203.340981:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234203.724944:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234203.724948:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234203.724954:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234203.724954:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234203.724958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234203.724959:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234203.725020:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234203.725025:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234203.725028:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234203.852961:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234203.852970:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234203.852972:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234203.852974:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234203.852976:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234203.852980:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234203.852985:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6672 00000100:00080000:1.0:1761234203.852988:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234203.852990:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.852992:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234203.852994:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.852995:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234203.852998:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234203.853000:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234203.853002:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234203.853004:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234203.853006:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234203.853007:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234203.853009:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853010:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234203.853011:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853012:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234203.853014:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234203.853016:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853017:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234203.853020:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234203.853022:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234203.853023:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853025:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234203.853027:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234203.853034:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761234203.853037:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234203.853040:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234203.853041:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853043:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234203.853045:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234203.853048:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234203.853057:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761234203.853062:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234203.853067:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234203.853074:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234203.853075:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234203.853078:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234203.853096:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853098:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234203.853099:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234203.853103:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234203.853104:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234203.853105:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234203.853106:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234203.853108:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234203.853110:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6672 00000100:00080000:1.0:1761234203.853112:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234203.853114:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853116:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234203.853118:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853120:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234203.853121:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234203.853122:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234203.853124:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234203.853126:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234203.853127:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234203.853129:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234203.853132:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853133:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234203.853134:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853136:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234203.853138:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234203.853140:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853143:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234203.853144:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234203.853166:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234203.853169:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853170:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234203.853173:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234203.853175:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853177:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234203.853180:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000020:00000040:1.0:1761234203.853182:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:3.0:1761234203.853182:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1761234203.853185:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234203.853186:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853187:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1761234203.853188:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234203.853189:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1761234203.853189:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234203.853190:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234203.853193:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:1.0:1761234203.853194:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761234203.853197:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234203.853200:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853201:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234203.853203:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853206:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1761234203.853206:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1761234203.853207:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234203.853209:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234203.853210:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761234203.853211:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003630282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232704:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234203.853215:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1761234203.853218:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853219:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 02000000:00000001:3.0:1761234203.853219:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1761234203.853221:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1761234203.853221:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234203.853223:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1761234203.853224:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234203.853225:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234203.853226:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234203.853227:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:3.0:1761234203.853227:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234203.853229:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853230:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234203.853231:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6672 00000100:00080000:1.0:1761234203.853233:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000010:3.0:1761234203.853235:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000100:00000040:1.0:1761234203.853236:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853237:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234203.853239:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853240:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234203.853242:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000400:00000010:3.0:1761234203.853242:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000020:00000040:1.0:1761234203.853243:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234203.853244:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234203.853246:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234203.853247:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000200:3.0:1761234203.853248:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232704, portal 25 00000020:00000001:1.0:1761234203.853249:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:3.0:1761234203.853250:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234203.853251:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039816128 : -104717669735488 : ffffa0c2847e63c0) 00000100:00000040:1.0:1761234203.853252:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853254:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234203.853255:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000040:3.0:1761234203.853256:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234228 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234203.853257:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234203.853259:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234203.853261:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853263:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853263:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 02000000:00000001:1.0:1761234203.853264:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234203.853266:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1761234203.853267:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 02000000:00000001:1.0:1761234203.853268:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234203.853269:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000001:1.0:1761234203.853270:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234203.853271:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000200:3.0:1761234203.853272:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245232704, offset 0 00000008:00000001:1.0:1761234203.853273:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853275:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234203.853277:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761234203.853279:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000200:3.0:1761234203.853279:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000001:1.0:1761234203.853281:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234203.853282:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234203.853283:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234203.853284:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234203.853285:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234203.853287:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761234203.853289:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234203.853292:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234203.853293:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:1.0:1761234203.853298:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234203.853299:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234203.853300:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000040:1.0:1761234203.853301:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234203.853304:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1761234203.853307:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234203.853307:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000001:1.0:1761234203.853309:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000200:3.0:1761234203.853311:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11853:11853:256:4294967295] 192.168.201.124@tcp LPNI seq info [11853:11853:8:4294967295] 00000100:00000040:1.0:1761234203.853312:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6682) 00000400:00000200:3.0:1761234203.853321:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234203.853327:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234203.853331:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234203.853338:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234203.853341:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761234203.853346:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234203.853349:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1761234203.853353:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234203.853356:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234203.853362:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234203.853367:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234203.853371:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234203.853373:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234203.853377:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234203.853380:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234228 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853389:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234203.853391:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234228 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853395:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853396:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234203.853402:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853403:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234203.853405:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853407:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234203.853409:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234203/real 1761234203] req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234203.853416:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234203.853417:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234203.853418:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234203.853419:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234203.853420:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234203.853422:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234203.853423:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234203.853427:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853431:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234203.853432:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853433:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234203.853434:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234203.853436:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853441:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234203.853443:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234203.853447:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234203.853449:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853449:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234203.853450:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853451:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234203.853452:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234203.853454:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234203.853456:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234203.853457:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234203.853459:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234203.853464:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003630282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232704:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234203.853467:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853469:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234203.853470:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000003630282c x1846784245232704/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234203.853474:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234203.853476:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234203.853477:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234203.853479:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234203.853480:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234203.853481:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234203.853482:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234203.853484:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a3ed3058. 02000000:00000001:3.0:1761234203.853487:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234203.853488:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234203.853489:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000003630282c. 00000100:00000001:3.0:1761234203.853491:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234203.853492:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853493:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853499:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234203.853500:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853502:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234203.853502:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234203.853503:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234203.853505:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853509:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234203.853510:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234203.853511:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234203.853512:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234203.853513:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d3898169 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232768:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234203.853515:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234203.853517:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234203.853517:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234203.853519:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234203.853521:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234203.853522:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234203.853523:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234203.853525:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234203.853527:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232768, portal 10 00000100:00000001:3.0:1761234203.853528:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234203.853529:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039796672 : -104717669754944 : ffffa0c2847e17c0) 00000100:00000040:3.0:1761234203.853533:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234228 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853537:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234203.853538:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234203.853539:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234203.853540:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245232768, offset 0 00000400:00000200:3.0:1761234203.853543:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234203.853546:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234203.853549:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234203.853550:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234203.853552:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234203.853553:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11854:11854:256:4294967295] 192.168.201.124@tcp LPNI seq info [11854:11854:8:4294967295] 00000400:00000200:3.0:1761234203.853559:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234203.853562:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234203.853563:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234203.853566:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234203.853567:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761234203.853570:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234203.853571:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1761234203.853573:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234203.853574:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234203.853576:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234203.853578:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234203.853580:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234203.853581:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234203.853582:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234203.853583:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234228 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853587:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234203.853588:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234228 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853592:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853593:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234203.853595:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853595:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234203.853596:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853597:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234203.853599:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234203/real 1761234203] req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234203.853604:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234203.853604:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234203.853605:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234203.853606:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234203.853607:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234203.853607:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234203.853609:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234203.853612:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853616:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234203.853616:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853617:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234203.853618:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234203.853620:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853624:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234203.853625:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234203.853629:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234203.853631:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853632:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234203.853632:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853633:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234203.853634:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234203.853635:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234203.853637:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234203.853638:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234203.853639:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234203.853643:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d3898169 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232768:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234203.853645:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234203.853646:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853650:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234203.853650:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234203.853652:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234203.853653:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234203.853654:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcae4007 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232832:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234203.853656:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234203.853656:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234203.853657:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234203.853658:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234203.853659:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234203.853660:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234203.853662:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234203.853663:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234203.853665:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232832, portal 4 00000100:00000001:3.0:1761234203.853666:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234203.853667:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039802752 : -104717669748864 : ffffa0c2847e2f80) 00000100:00000040:3.0:1761234203.853669:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234228 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853673:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234203.853675:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234203.853676:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234203.853677:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245232832, offset 0 00000400:00000200:3.0:1761234203.853679:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234203.853682:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234203.853683:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234203.853685:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234203.853686:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234203.853687:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11855:11855:256:4294967295] 192.168.201.124@tcp LPNI seq info [11855:11855:8:4294967295] 00000400:00000200:3.0:1761234203.853692:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234203.853695:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234203.853696:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234203.853699:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234203.853699:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1761234203.853702:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234203.853704:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1761234203.853706:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234203.853707:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234203.853708:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234203.853710:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234203.853712:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234203.853713:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234203.853714:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234203.853715:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234228 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853718:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234203.853720:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234228 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853724:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853725:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234203.853726:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853727:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234203.853728:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853729:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234203.853730:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234203/real 1761234203] req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234203.853734:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234203.853734:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234203.853735:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234203.853736:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234203.853737:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234203.853738:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234203.853739:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234203.853743:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853746:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234203.853747:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853748:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234203.853748:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234203.853750:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234203.853753:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234203.853754:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234203.853758:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234203.853759:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853759:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234203.853761:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853761:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234203.853762:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234203.853764:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234203.853765:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234203.853766:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234203.853768:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234203.853771:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcae4007 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232832:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234203.853773:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853774:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234203.853775:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000d3898169 x1846784245232768/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234203.853778:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234203.853779:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234203.853780:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234203.853781:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234203.853781:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234203.853782:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234203.853783:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234203.853784:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:3.0:1761234203.853785:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234203.853786:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234203.853786:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000d3898169. 00000100:00000001:3.0:1761234203.853788:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234203.853788:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853789:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234203.853791:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fcae4007 x1846784245232832/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234228 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234203.853794:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234203.853795:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234203.853795:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234203.853798:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234203.853798:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234203.853799:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234203.853800:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234203.853801:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:3.0:1761234203.853802:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234203.853802:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234203.853803:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fcae4007. 00000100:00000001:3.0:1761234203.853804:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234203.853805:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853806:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234203.853807:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234203.853808:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234203.853809:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234203.853810:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234204.364953:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234204.364958:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234204.364961:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234204.364966:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234204.364972:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234204.364976:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234204.364979:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234204.364982:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234204.364985:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234204.749016:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234204.749021:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234204.749028:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234204.749029:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234204.749033:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234204.749034:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234204.749106:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234204.749108:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234204.749111:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234205.388942:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234205.388947:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234205.388949:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234205.388952:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234205.388956:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234205.388959:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234205.388961:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234205.388963:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234205.388965:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234205.772958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234205.772963:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234205.772970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234205.772971:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234205.772976:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234205.772978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234205.773103:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234205.773106:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234205.773110:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234206.412929:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234206.412933:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234206.412934:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234206.412937:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234206.412941:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234206.412943:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234206.412945:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234206.412947:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234206.412948:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234206.796959:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234206.796964:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234206.796969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234206.796970:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234206.796973:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234206.796974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234206.797044:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234206.797048:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234206.797052:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234207.436998:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234207.437005:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234207.437007:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234207.437011:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234207.437016:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234207.437019:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234207.437022:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234207.437024:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234207.437027:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234207.820959:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234207.820965:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234207.820972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234207.820972:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234207.820975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234207.820976:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234207.821070:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234207.821087:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234207.821092:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234208.460964:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234208.460970:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234208.460971:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234208.460976:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234208.460982:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234208.460986:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234208.460988:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234208.460991:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234208.460993:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234208.845018:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234208.845026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234208.845034:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234208.845036:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234208.845039:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234208.845040:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234208.845135:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234208.845139:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234208.845142:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234208.972962:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234208.972971:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234208.972972:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234208.972974:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234208.972977:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234208.972981:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234208.972986:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6677 00000100:00080000:1.0:1761234208.972989:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234208.972991:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.972992:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234208.972994:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.972995:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234208.972998:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234208.972999:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234208.973001:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234208.973004:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234208.973005:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234208.973007:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234208.973009:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973010:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234208.973011:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973012:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234208.973013:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234208.973016:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973017:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234208.973019:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234208.973021:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234208.973023:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973024:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234208.973026:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234208.973034:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f8a85602. 00000020:00000040:1.0:1761234208.973036:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234208.973039:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234208.973040:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973042:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234208.973045:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234208.973048:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234208.973078:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761234208.973083:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234208.973088:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234208.973095:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234208.973097:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234208.973099:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f8a85602] to pc [ptlrpcd_rcv+-1] req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234208.973116:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973117:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234208.973118:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234208.973121:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234208.973121:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234208.973122:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234208.973123:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234208.973125:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234208.973126:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6677 00000100:00080000:1.0:1761234208.973127:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234208.973128:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973129:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234208.973130:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973131:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234208.973132:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234208.973133:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234208.973134:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234208.973135:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234208.973136:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234208.973137:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234208.973138:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973140:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234208.973141:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973142:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234208.973143:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234208.973145:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973146:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234208.973148:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234208.973152:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234208.973153:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973154:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234208.973157:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234208.973159:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973160:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234208.973164:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fcae4007. 00000020:00000040:1.0:1761234208.973165:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:3.0:1761234208.973166:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1761234208.973167:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234208.973170:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973171:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1761234208.973171:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1761234208.973173:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1761234208.973174:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234208.973175:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234208.973179:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000040:3.0:1761234208.973181:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234208.973182:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234208.973188:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973197:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234208.973198:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234208.973199:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973200:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234208.973204:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fcae4007] to pc [ptlrpcd_rcv+-1] req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234208.973205:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234208.973207:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973212:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234208.973213:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f8a85602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232896:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761234208.973214:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234208.973215:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234208.973218:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1761234208.973219:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234208.973221:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234208.973222:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1761234208.973222:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1761234208.973223:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1761234208.973225:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973226:0:468:0:(import.c:519:import_select_connection()) Process entered 02000000:00000001:3.0:1761234208.973229:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1761234208.973230:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6677 00000100:00080000:1.0:1761234208.973232:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234208.973234:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973235:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000010:3.0:1761234208.973235:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 00000100:00000040:1.0:1761234208.973237:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 02000000:00000001:3.0:1761234208.973239:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973240:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234208.973242:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234208.973243:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234208.973244:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234208.973245:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000400:00000010:3.0:1761234208.973245:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000020:00000001:1.0:1761234208.973246:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234208.973249:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234208.973251:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973252:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1761234208.973253:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000040:1.0:1761234208.973254:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234208.973257:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234208.973258:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234208.973259:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000200:3.0:1761234208.973260:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232896, portal 25 00000100:00000001:1.0:1761234208.973261:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234208.973264:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1761234208.973264:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1761234208.973266:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234208.973267:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039819776 : -104717669731840 : ffffa0c2847e7200) 02000000:00000001:1.0:1761234208.973271:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973273:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234208.973275:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000040:3.0:1761234208.973275:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234233 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000008:00000001:1.0:1761234208.973279:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973282:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234208.973286:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000d3898169. 00000100:00000001:3.0:1761234208.973287:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000040:1.0:1761234208.973289:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234208.973293:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000040:3.0:1761234208.973293:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 02000000:00000001:1.0:1761234208.973295:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973297:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:3.0:1761234208.973297:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 02000000:00000001:1.0:1761234208.973299:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234208.973300:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1761234208.973301:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245232896, offset 0 02000000:00000010:1.0:1761234208.973304:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234208.973308:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234208.973310:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00100000:1.0:1761234208.973312:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234208.973317:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234208.973318:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234208.973321:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000d3898169] to pc [ptlrpcd_rcv+-1] req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234208.973324:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:1.0:1761234208.973330:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234208.973331:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234208.973334:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6688) 00000400:00000200:3.0:1761234208.973334:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234208.973338:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234208.973341:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234208.973344:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11856:11856:256:4294967295] 192.168.201.124@tcp LPNI seq info [11856:11856:8:4294967295] 00000400:00000200:3.0:1761234208.973351:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234208.973357:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234208.973361:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234208.973366:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234208.973368:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1761234208.973372:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234208.973374:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1761234208.973377:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234208.973379:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234208.973384:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234208.973387:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234208.973390:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234208.973392:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234208.973395:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234208.973397:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234233 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973403:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234208.973404:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234233 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973408:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973409:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234208.973414:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973415:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234208.973417:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973418:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234208.973420:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234208/real 1761234208] req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234208.973426:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234208.973427:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234208.973428:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234208.973429:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234208.973430:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234208.973431:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234208.973432:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234208.973436:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973440:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234208.973441:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973443:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234208.973445:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234208.973446:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973450:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234208.973452:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234208.973460:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234208.973462:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973463:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234208.973464:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973465:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234208.973466:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234208.973468:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234208.973469:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234208.973471:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234208.973473:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234208.973477:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f8a85602 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245232896:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234208.973479:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973480:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234208.973481:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f8a85602 x1846784245232896/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234208.973485:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234208.973487:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234208.973488:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234208.973491:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234208.973492:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234208.973493:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234208.973494:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234208.973495:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f654abd8. 02000000:00000001:3.0:1761234208.973498:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234208.973499:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234208.973500:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f8a85602. 00000100:00000001:3.0:1761234208.973502:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234208.973503:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973504:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973510:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234208.973511:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973512:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234208.973513:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234208.973514:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234208.973515:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973519:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234208.973520:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234208.973522:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234208.973523:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234208.973525:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fcae4007 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232960:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234208.973529:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234208.973530:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234208.973531:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234208.973533:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234208.973535:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234208.973536:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234208.973539:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234208.973542:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234208.973544:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245232960, portal 10 00000100:00000001:3.0:1761234208.973546:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234208.973547:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039802752 : -104717669748864 : ffffa0c2847e2f80) 00000100:00000040:3.0:1761234208.973552:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234233 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973559:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234208.973561:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234208.973563:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234208.973565:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245232960, offset 0 00000400:00000200:3.0:1761234208.973569:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234208.973574:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234208.973578:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234208.973580:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234208.973582:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234208.973584:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11857:11857:256:4294967295] 192.168.201.124@tcp LPNI seq info [11857:11857:8:4294967295] 00000400:00000200:3.0:1761234208.973590:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234208.973593:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234208.973594:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234208.973597:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234208.973599:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1761234208.973601:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234208.973603:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1761234208.973605:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234208.973606:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234208.973608:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234208.973610:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234208.973611:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234208.973612:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234208.973614:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234208.973616:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234233 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973620:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234208.973622:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234233 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973625:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973626:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234208.973628:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973629:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234208.973630:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973632:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234208.973633:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234208/real 1761234208] req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234208.973638:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234208.973638:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234208.973639:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234208.973640:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234208.973641:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234208.973642:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234208.973644:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234208.973648:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973652:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234208.973652:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973653:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234208.973654:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234208.973655:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973659:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234208.973660:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234208.973663:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234208.973665:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973665:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234208.973666:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973667:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234208.973668:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234208.973669:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234208.973671:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234208.973672:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234208.973674:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234208.973678:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fcae4007 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245232960:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234208.973680:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234208.973681:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973684:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234208.973685:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234208.973687:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234208.973687:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234208.973689:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000d3898169 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233024:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234208.973691:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234208.973692:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234208.973692:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234208.973693:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234208.973695:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234208.973696:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234208.973697:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234208.973699:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234208.973700:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233024, portal 4 00000100:00000001:3.0:1761234208.973702:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234208.973702:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039796672 : -104717669754944 : ffffa0c2847e17c0) 00000100:00000040:3.0:1761234208.973705:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234233 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973709:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234208.973710:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234208.973711:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234208.973713:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245233024, offset 0 00000400:00000200:3.0:1761234208.973715:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234208.973717:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234208.973719:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234208.973721:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234208.973722:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234208.973723:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11858:11858:256:4294967295] 192.168.201.124@tcp LPNI seq info [11858:11858:8:4294967295] 00000400:00000200:3.0:1761234208.973728:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234208.973731:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234208.973732:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234208.973734:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234208.973735:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1761234208.973738:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234208.973739:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 6 secs later) 00000800:00000100:3.0:1761234208.973741:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234208.973742:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234208.973744:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234208.973746:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234208.973747:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234208.973748:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234208.973749:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234208.973750:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234233 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973754:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234208.973755:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234233 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973758:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973759:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234208.973760:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973761:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234208.973762:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973763:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234208.973765:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234208/real 1761234208] req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234208.973769:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234208.973769:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234208.973770:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234208.973771:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234208.973772:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234208.973772:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234208.973773:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234208.973777:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973780:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234208.973781:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973782:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234208.973783:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234208.973785:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234208.973789:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234208.973790:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234208.973793:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234208.973794:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973795:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234208.973796:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973797:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234208.973797:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234208.973798:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234208.973799:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234208.973800:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234208.973802:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234208.973806:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000d3898169 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233024:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234208.973807:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973808:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234208.973809:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fcae4007 x1846784245232960/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234208.973813:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234208.973814:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234208.973814:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234208.973816:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234208.973817:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234208.973817:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234208.973819:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234208.973820:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:3.0:1761234208.973821:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234208.973822:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234208.973823:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fcae4007. 00000100:00000001:3.0:1761234208.973825:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234208.973825:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973826:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234208.973827:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000d3898169 x1846784245233024/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234233 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234208.973831:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234208.973832:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234208.973832:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234208.973834:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234208.973834:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234208.973835:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234208.973836:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234208.973836:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:3.0:1761234208.973837:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234208.973838:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234208.973838:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000d3898169. 00000100:00000001:3.0:1761234208.973839:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234208.973840:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973841:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234208.973862:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234208.973863:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234208.973864:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234208.973865:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234209.484965:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234209.484969:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234209.484970:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234209.484974:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234209.484978:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234209.484981:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234209.484983:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234209.484985:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234209.484986:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234209.868953:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234209.868959:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234209.868966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234209.868967:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234209.868971:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234209.868972:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234209.869084:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234209.869087:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234209.869089:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234210.508918:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234210.508922:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234210.508923:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234210.508927:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234210.508931:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234210.508934:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234210.508936:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234210.508937:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234210.508939:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893034:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234210.893044:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234210.893046:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234210.893049:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893058:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234210.893061:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893065:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893068:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893073:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234210.893075:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234210.893077:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234210.893084:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234210.893089:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234210.893089:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893092:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234210.893094:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893096:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893098:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893103:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234210.893105:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234210.893106:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234210.893110:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234210.893113:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234210.893114:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893116:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234210.893118:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893120:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234210.893123:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234210.893152:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234210.893157:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234210.893162:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234211.532968:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234211.532973:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234211.532974:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234211.532977:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234211.532981:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234211.532984:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234211.532985:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234211.532987:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234211.532989:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234211.916928:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234211.916932:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234211.916937:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234211.916938:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234211.916940:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234211.916941:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234211.917050:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234211.917055:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234211.917059:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234212.556997:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234212.557005:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234212.557007:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234212.557012:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234212.557018:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234212.557022:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234212.557026:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234212.557028:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234212.557031:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234212.941090:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234212.941095:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234212.941101:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234212.941102:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234212.941105:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234212.941106:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234212.941202:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234212.941205:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234212.941208:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234213.581041:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234213.581046:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234213.581048:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234213.581053:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234213.581059:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234213.581063:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234213.581066:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234213.581068:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234213.581071:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234213.964904:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234213.964909:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234213.964914:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234213.964916:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234213.964920:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234213.964921:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234213.965000:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234213.965003:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234213.965005:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234214.093023:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234214.093035:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234214.093037:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234214.093040:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234214.093042:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234214.093048:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234214.093055:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6683 00000100:00080000:1.0:1761234214.093058:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234214.093060:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093062:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234214.093064:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093066:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234214.093069:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234214.093073:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234214.093076:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234214.093079:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234214.093081:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234214.093083:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234214.093085:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093086:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234214.093087:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093088:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234214.093089:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234214.093092:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093094:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234214.093096:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234214.093099:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234214.093101:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234214.093103:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234214.093104:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234214.093116:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003630282c. 00000020:00000040:1.0:1761234214.093119:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234214.093122:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234214.093123:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234214.093127:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234214.093129:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234214.093133:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234214.093141:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761234214.093148:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234214.093156:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234214.093164:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234214.093166:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234214.093169:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003630282c] to pc [ptlrpcd_rcv+-1] req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234214.093188:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234214.093190:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234214.093192:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234214.093195:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234214.093196:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234214.093197:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234214.093197:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234214.093199:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234214.093200:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6683 00000100:00080000:1.0:1761234214.093201:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234214.093203:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093204:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234214.093205:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093206:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234214.093208:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234214.093208:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234214.093209:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234214.093211:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234214.093212:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234214.093213:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234214.093214:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093215:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234214.093216:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093217:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234214.093218:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234214.093219:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093220:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234214.093221:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234214.093227:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234214.093228:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234214.093229:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234214.093233:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234214.093236:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234214.093238:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234214.093240:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f4f489ba. 00000020:00000040:1.0:1761234214.093241:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234214.093243:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234214.093244:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234214.093245:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234214.093246:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234214.093247:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234214.093252:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234214.093255:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234214.093257:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234214.093261:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234214.093263:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234214.093264:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f4f489ba] to pc [ptlrpcd_rcv+-1] req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234214.093269:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234214.093270:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234214.093271:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234214.093273:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234214.093273:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234214.093275:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234214.093276:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234214.093277:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234214.093278:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234214.093283:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6683 00000100:00000001:3.0:1761234214.093284:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1761234214.093286:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1761234214.093287:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1761234214.093290:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093293:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234214.093297:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234214.093298:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000040:3.0:1761234214.093301:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000020:00000001:1.0:1761234214.093303:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234214.093304:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234214.093305:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234214.093306:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234214.093313:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234214.093319:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234214.093326:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234214.093328:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234214.093329:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1761234214.093331:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234214.093335:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 02000000:00000001:3.0:1761234214.093337:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1761234214.093338:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234214.093343:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 02000000:00000001:3.0:1761234214.093344:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761234214.093347:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00100000:3.0:1761234214.093350:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003630282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233088:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761234214.093353:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234214.093359:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234214.093361:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234214.093362:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761234214.093366:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234214.093367:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1761234214.093368:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234214.093372:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1761234214.093373:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1761234214.093378:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234214.093379:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1761234214.093382:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234214.093385:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c3b47c4a. 00000020:00000040:1.0:1761234214.093387:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234214.093389:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000010:3.0:1761234214.093389:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000009baf651a. 02000000:00000001:1.0:1761234214.093391:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234214.093393:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234214.093394:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234214.093398:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234214.093401:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234214.093402:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 02000000:00000010:1.0:1761234214.093405:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761234214.093409:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234214.093409:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00100000:1.0:1761234214.093419:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234214.093420:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233088, portal 25 00000100:00000001:3.0:1761234214.093430:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1761234214.093433:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234214.093434:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039816128 : -104717669735488 : ffffa0c2847e63c0) 00000100:00000001:1.0:1761234214.093435:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234214.093441:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c3b47c4a] to pc [ptlrpcd_rcv+-1] req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:3.0:1761234214.093451:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234239 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234214.093453:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234214.093456:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234214.093460:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6693) 00000100:00000001:3.0:1761234214.093468:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234214.093474:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234214.093477:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234214.093481:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245233088, offset 0 00000400:00000200:3.0:1761234214.093492:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234214.093507:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234214.093513:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234214.093516:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234214.093518:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234214.093521:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11859:11859:256:4294967295] 192.168.201.124@tcp LPNI seq info [11859:11859:8:4294967295] 00000400:00000200:3.0:1761234214.093530:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234214.093534:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234214.093538:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234214.093544:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234214.093546:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761234214.093551:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234214.093553:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1761234214.093556:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234214.093558:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234214.093562:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234214.093565:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234214.093568:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234214.093570:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234214.093573:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234214.093576:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234239 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093583:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234214.093585:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234239 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093589:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.093590:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234214.093595:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.093596:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234214.093597:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.093599:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234214.093601:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234214/real 1761234214] req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234214.093607:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234214.093608:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234214.093609:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234214.093610:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234214.093611:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234214.093612:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234214.093614:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234214.093618:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093622:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234214.093623:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093625:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234214.093626:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234214.093627:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093631:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234214.093634:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234214.093638:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234214.093639:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093640:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234214.093641:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093643:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234214.093644:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234214.093646:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234214.093648:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234214.093650:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234214.093652:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234214.093657:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003630282c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233088:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234214.093659:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234214.093662:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093668:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234214.093669:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234214.093671:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234214.093673:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234214.093675:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f4f489ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233152:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234214.093679:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234214.093680:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234214.093682:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234214.093683:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234214.093686:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234214.093687:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234214.093690:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234214.093693:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234214.093695:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233152, portal 10 00000100:00000001:3.0:1761234214.093697:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234214.093698:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039793024 : -104717669758592 : ffffa0c2847e0980) 00000100:00000040:3.0:1761234214.093703:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234239 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093710:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234214.093712:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234214.093714:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234214.093716:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245233152, offset 0 00000400:00000200:3.0:1761234214.093720:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234214.093724:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234214.093728:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234214.093730:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234214.093732:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234214.093734:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11860:11860:256:4294967295] 192.168.201.124@tcp LPNI seq info [11860:11860:8:4294967295] 00000400:00000200:3.0:1761234214.093739:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234214.093742:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234214.093744:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234214.093747:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234214.093748:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761234214.093751:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234214.093753:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1761234214.093755:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234214.093756:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234214.093758:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234214.093760:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234214.093761:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234214.093762:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234214.093764:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234214.093765:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234239 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093770:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234214.093771:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234239 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093775:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.093776:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234214.093779:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.093780:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234214.093782:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.093784:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234214.093786:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234214/real 1761234214] req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234214.093793:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234214.093795:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234214.093796:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234214.093798:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234214.093799:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234214.093801:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234214.093803:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234214.093810:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093815:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234214.093817:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093818:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234214.093820:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234214.093822:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093829:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234214.093832:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234214.093838:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234214.093841:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093841:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234214.093880:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093881:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234214.093883:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234214.093884:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234214.093886:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234214.093887:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234214.093890:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234214.093895:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f4f489ba pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233152:192.168.201.124@tcp:38: 00000100:00000001:3.0:1761234214.093897:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093899:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234214.093900:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000003630282c x1846784245233088/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234214.093904:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234214.093906:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234214.093908:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000009baf651a. 02000000:00000001:3.0:1761234214.093910:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234214.093911:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234214.093912:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:3.0:1761234214.093914:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234214.093916:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a174ebf2. 02000000:00000001:3.0:1761234214.093921:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234214.093922:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234214.093923:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000003630282c. 00000100:00000001:3.0:1761234214.093925:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234214.093926:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093927:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234214.093928:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f4f489ba x1846784245233152/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234214.093932:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234214.093932:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234214.093933:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234214.093935:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234214.093935:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234214.093936:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:3.0:1761234214.093937:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234214.093937:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:3.0:1761234214.093939:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234214.093939:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234214.093940:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f4f489ba. 00000100:00000001:3.0:1761234214.093941:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234214.093942:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093943:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.093949:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234214.093950:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.093952:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234214.093969:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234214.093970:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234214.093973:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.093978:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234214.093979:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234214.093980:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1761234214.093981:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234214.093983:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c3b47c4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233216:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234214.093985:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1761234214.093986:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234214.093987:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234214.093988:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234214.093990:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234214.093991:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1761234214.093993:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000e1151e64. 00000400:00000010:3.0:1761234214.093995:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000080182d66. 00000100:00000200:3.0:1761234214.093997:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233216, portal 4 00000100:00000001:3.0:1761234214.093998:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234214.093999:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039818560 : -104717669733056 : ffffa0c2847e6d40) 00000100:00000040:3.0:1761234214.094002:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234239 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.094006:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:3.0:1761234214.094007:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:3.0:1761234214.094008:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000b76f86a3. 00000100:00000200:3.0:1761234214.094009:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245233216, offset 0 00000400:00000200:3.0:1761234214.094012:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:3.0:1761234214.094015:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:3.0:1761234214.094017:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:3.0:1761234214.094019:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1761234214.094020:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:3.0:1761234214.094022:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11861:11861:256:4294967295] 192.168.201.124@tcp LPNI seq info [11861:11861:8:4294967295] 00000400:00000200:3.0:1761234214.094027:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:3.0:1761234214.094030:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234214.094032:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000003bd24964 (tot 1676530). 00000800:00000200:3.0:1761234214.094034:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.0:1761234214.094036:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1761234214.094038:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:3.1:1761234214.094040:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 1 secs later) 00000800:00000100:3.0:1761234214.094043:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:3.0:1761234214.094044:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000003bd24964 (tot 1676282). 00000400:00000200:3.0:1761234214.094045:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1761234214.094047:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1761234214.094050:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000b76f86a3 00000400:00000010:3.0:1761234214.094051:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000b76f86a3. 00000100:00000001:3.0:1761234214.094052:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234214.094054:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234239 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.094058:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234214.094059:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234239 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.094063:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.094063:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:3.0:1761234214.094065:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.094066:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1761234214.094067:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.094068:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1761234214.094070:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234214/real 1761234214] req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:3.0:1761234214.094074:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1761234214.094075:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1761234214.094076:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000e1151e64. 00000400:00000200:3.0:1761234214.094077:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000080182d66 00000400:00000010:3.0:1761234214.094078:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000080182d66. 00000100:00000001:3.0:1761234214.094079:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:3.0:1761234214.094080:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234214.094084:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.094088:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:3.0:1761234214.094088:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.094089:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234214.094090:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1761234214.094091:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234214.094095:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1761234214.094097:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234214.094101:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1761234214.094102:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.094103:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1761234214.094103:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.094104:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1761234214.094105:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1761234214.094106:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1761234214.094108:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:3.0:1761234214.094109:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1761234214.094110:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:3.0:1761234214.094114:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c3b47c4a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233216:192.168.201.124@tcp:8: 00000100:00000001:3.0:1761234214.094116:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.094117:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234214.094118:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000c3b47c4a x1846784245233216/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234239 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:3.0:1761234214.094122:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1761234214.094122:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1761234214.094123:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000071d5cb10. 02000000:00000001:3.0:1761234214.094125:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1761234214.094125:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:3.0:1761234214.094126:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:3.0:1761234214.094127:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:3.0:1761234214.094127:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:3.0:1761234214.094129:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1761234214.094130:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1761234214.094130:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000c3b47c4a. 00000100:00000001:3.0:1761234214.094132:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1761234214.094133:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.094134:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234214.094136:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234214.094137:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234214.094138:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234214.094139:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234214.605071:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234214.605078:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234214.605081:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234214.605086:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234214.605092:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234214.605097:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234214.605100:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234214.605103:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234214.605106:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234214.989011:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234214.989016:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234214.989024:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234214.989025:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234214.989029:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234214.989031:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234214.989091:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234214.989094:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234214.989097:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234215.629007:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234215.629013:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234215.629015:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234215.629019:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234215.629025:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234215.629028:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234215.629030:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234215.629033:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234215.629035:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234216.012950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234216.012955:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234216.012963:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234216.012964:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234216.012967:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234216.012968:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234216.013060:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234216.013064:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234216.013067:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234216.653006:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234216.653011:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234216.653013:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234216.653018:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234216.653023:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234216.653026:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234216.653029:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234216.653031:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234216.653033:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234217.036995:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234217.037000:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234217.037007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234217.037009:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234217.037013:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234217.037015:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234217.037073:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234217.037076:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234217.037078:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234217.677019:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234217.677024:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234217.677026:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234217.677030:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234217.677035:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234217.677039:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234217.677041:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234217.677044:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234217.677046:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234218.061006:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234218.061010:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234218.061015:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234218.061016:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234218.061018:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234218.061019:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234218.061086:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234218.061089:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234218.061091:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234218.701048:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234218.701054:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234218.701055:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234218.701059:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234218.701064:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234218.701068:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234218.701070:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234218.701072:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234218.701074:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234219.085018:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234219.085024:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234219.085032:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234219.085034:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234219.085037:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234219.085038:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234219.085130:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234219.085134:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234219.085137:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234219.213063:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234219.213078:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234219.213079:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234219.213082:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234219.213085:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234219.213107:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234219.213115:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6688 00000100:00080000:3.0:1761234219.213118:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234219.213121:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213123:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234219.213125:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213127:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234219.213130:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234219.213132:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234219.213136:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234219.213139:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234219.213141:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234219.213143:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234219.213146:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213147:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234219.213148:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213149:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234219.213167:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234219.213170:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213172:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234219.213174:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234219.213177:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234219.213180:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234219.213182:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234219.213183:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234219.213194:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000108ab0a5. 00000020:00000040:3.0:1761234219.213197:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234219.213201:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234219.213203:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234219.213206:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234219.213209:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234219.213213:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234219.213222:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000071d5cb10. 00000100:00000001:3.0:1761234219.213229:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234219.213235:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234219.213259:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234219.213262:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234219.213265:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000108ab0a5] to pc [ptlrpcd_rcv+-1] req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234219.213290:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234219.213293:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234219.213295:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234219.213298:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234219.213299:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234219.213300:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234219.213301:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234219.213303:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234219.213305:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6688 00000100:00080000:3.0:1761234219.213306:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234219.213308:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213309:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234219.213310:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213312:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234219.213314:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234219.213315:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234219.213316:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234219.213318:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234219.213320:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234219.213321:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234219.213324:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213325:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234219.213326:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213327:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234219.213329:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234219.213331:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213332:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234219.213333:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234219.213340:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234219.213341:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234219.213344:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234219.213349:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234219.213352:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234219.213354:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234219.213356:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002431276f. 00000020:00000040:3.0:1761234219.213358:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:2.0:1761234219.213362:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1761234219.213362:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234219.213370:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.213373:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234219.213374:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234219.213382:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1761234219.213382:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234219.213388:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234219.213394:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000009baf651a. 00000100:00000001:3.0:1761234219.213397:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234219.213399:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:3.0:1761234219.213404:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234219.213457:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234219.213459:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.213462:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1761234219.213464:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002431276f] to pc [ptlrpcd_rcv+-1] req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234219.213466:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234219.213474:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1761234219.213476:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234219.213478:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 02000000:00000001:2.0:1761234219.213479:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1761234219.213480:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234219.213486:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234219.213488:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234219.213490:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234219.213491:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234219.213493:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234219.213495:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6688 00000100:00080000:3.0:1761234219.213498:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1761234219.213499:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233280:192.168.201.124@tcp:250: 00000100:00000040:3.0:1761234219.213502:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213506:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234219.213509:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213513:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:2.0:1761234219.213517:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1761234219.213517:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234219.213520:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761234219.213522:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234219.213524:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 02000000:00000001:2.0:1761234219.213525:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1761234219.213526:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:2.0:1761234219.213530:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234219.213530:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:2.0:1761234219.213536:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1761234219.213537:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213540:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234219.213541:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213542:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234219.213543:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 02000000:00000010:2.0:1761234219.213545:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062f3eded. 00000100:00080000:3.0:1761234219.213545:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213547:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234219.213549:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234219.213550:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234219.213554:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1761234219.213555:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 02000000:00000001:3.0:1761234219.213558:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234219.213561:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234219.213563:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000010:2.0:1761234219.213565:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000008:00000001:3.0:1761234219.213578:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234219.213581:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233280, portal 25 00000100:00000001:3.0:1761234219.213581:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.213588:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234219.213589:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091110144 : -104717618441472 : ffffa0c2878d1300) 00000100:00000010:3.0:1761234219.213603:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f61de4c8. 00000100:00000040:2.0:1761234219.213607:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234244 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000020:00000040:3.0:1761234219.213607:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:3.0:1761234219.213617:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234219.213618:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234219.213622:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234219.213623:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234219.213626:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.213627:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 02000000:00000010:3.0:1761234219.213631:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000df51b12a. 00000100:00000040:2.0:1761234219.213633:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:3.0:1761234219.213636:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234219.213638:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000007b9e8310. 00000100:00100000:3.0:1761234219.213642:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234219.213643:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245233280, offset 0 00000100:00000001:3.0:1761234219.213655:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234219.213657:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234219.213659:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000040:3.0:1761234219.213662:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f61de4c8] to pc [ptlrpcd_rcv+-1] req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234219.213668:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234219.213669:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:3.0:1761234219.213675:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6698) 00000400:00000200:2.0:1761234219.213676:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234219.213683:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234219.213687:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234219.213690:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234219.213694:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11862:11862:256:4294967295] 192.168.201.124@tcp LPNI seq info [11862:11862:8:4294967295] 00000400:00000200:2.0:1761234219.213707:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234219.213711:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234219.213715:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000aee9c1bf (tot 1676530). 00000800:00000200:2.0:1761234219.213721:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234219.213725:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:2.0:1761234219.213746:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.213748:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234219.213750:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.213753:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.213754:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234219.213760:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234219.213762:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1761234219.213763:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234219.213765:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234219.213766:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234219.213768:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234219.213774:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234219.213775:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234219.213777:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234219.213778:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234219.213780:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233344:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234219.213783:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234219.213784:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234219.213785:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234219.213786:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234219.213788:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234219.213790:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234219.213792:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000ccddd6a4. 00000400:00000010:2.0:1761234219.213795:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000001d64cf84. 00000100:00000200:2.0:1761234219.213797:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233344, portal 10 00000100:00000001:2.0:1761234219.213798:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234219.213798:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091128384 : -104717618423232 : ffffa0c2878d5a40) 00000100:00000040:2.0:1761234219.213803:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234244 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234219.213807:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234219.213809:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234219.213811:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000000f27192d. 00000100:00000200:2.0:1761234219.213813:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245233344, offset 0 00000400:00000200:2.0:1761234219.213818:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234219.213821:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234219.213823:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234219.213825:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234219.213826:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234219.213828:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11863:11863:255:4294967295] 192.168.201.124@tcp LPNI seq info [11863:11863:7:4294967295] 00000400:00000200:2.0:1761234219.213834:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234219.213837:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234219.213840:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676778). 00000800:00000200:2.0:1761234219.213842:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000400:00000200:0.0:1761234219.213902:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000800:00000200:2.1:1761234219.213906:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:2.0:1761234219.213910:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.213911:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234219.213913:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.213915:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234219.213929:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234219.213934:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234219.213935:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234219.213937:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234219.213938:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234219.213940:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233408:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234219.213943:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234219.213944:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234219.213945:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234219.213946:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234219.213949:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000116ca75c. 02000000:00000001:2.0:1761234219.213950:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234219.213952:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000003966d2af. 00000400:00000010:2.0:1761234219.213954:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234219.213956:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233408, portal 4 00000100:00000001:2.0:1761234219.213958:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234219.213958:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091124736 : -104717618426880 : ffffa0c2878d4c00) 00000100:00000040:2.0:1761234219.213962:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234244 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234219.213966:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234219.213970:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000200:0.0:1761234219.213972:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000400:00000010:2.0:1761234219.213973:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000003f47b3af. 00000100:00000200:2.0:1761234219.213979:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245233408, offset 0 00000400:00000200:2.0:1761234219.213985:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234219.213992:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234219.213994:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234219.213996:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234219.213999:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234219.214002:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11864:11864:254:4294967295] 192.168.201.124@tcp LPNI seq info [11864:11864:6:4294967295] 00000400:00000200:0.0:1761234219.214004:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000400:00000200:2.0:1761234219.214019:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234219.214023:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234219.214029:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009b62b3c1 (tot 1677026). 00000800:00000200:2.0:1761234219.214031:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234219.214034:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:2.0:1761234219.214036:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.214037:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234219.214038:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.214040:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.214041:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234219.214043:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234219.214044:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1761234219.214046:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234219.214047:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234219.214048:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234219.214049:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234219.724961:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234219.724966:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234219.724967:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234219.724971:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234219.724975:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234219.724978:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234219.724979:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234219.724981:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234219.724983:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234220.108990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234220.108995:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234220.109000:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234220.109002:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234220.109005:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234220.109007:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234220.109095:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234220.109099:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234220.109103:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234220.748961:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234220.748965:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234220.748966:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234220.748969:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234220.748974:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234220.748977:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234220.748979:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234220.748981:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234220.748983:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.132976:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234221.132987:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234221.132989:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234221.132992:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.132997:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234221.132999:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133001:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133003:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234221.133009:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234221.133010:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234221.133014:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234221.133016:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234221.133017:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133019:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234221.133020:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133021:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133023:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234221.133027:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234221.133028:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234221.133029:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234221.133030:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234221.133030:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133032:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234221.133033:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133035:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234221.133036:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234221.133099:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234221.133102:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234221.133105:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234221.772950:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234221.772956:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234221.772957:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234221.772962:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234221.772967:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234221.772970:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234221.772973:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234221.772974:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234221.772977:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234222.157000:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234222.157005:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234222.157012:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234222.157014:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234222.157018:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234222.157019:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234222.157098:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234222.157102:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234222.157105:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1761234222.285092:0:88982:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:0.0:1761234222.285132:0:88982:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1761234222.285138:0:88982:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:0.0:1761234222.285147:0:88982:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761234222.285150:0:88982:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000aee9c1bf (tot 1676778). 00000400:00000200:0.0:1761234222.285156:0:88982:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761234222.285159:0:88982:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761234222.285163:0:88982:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000007b9e8310 00000400:00000010:0.0:1761234222.285165:0:88982:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000007b9e8310. 00000100:00000001:0.0:1761234222.285169:0:88982:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761234222.285174:0:88982:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234244 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234222.285193:0:88982:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761234222.285195:0:88982:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234244 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234222.285200:0:88982:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234222.285201:0:88982:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761234222.285209:0:88982:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761234222.285211:0:88982:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676530). 00000400:00000200:0.0:1761234222.285212:0:88982:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761234222.285214:0:88982:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761234222.285217:0:88982:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000000f27192d 00000400:00000010:0.0:1761234222.285218:0:88982:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000000f27192d. 00000100:00000001:0.0:1761234222.285219:0:88982:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761234222.285221:0:88982:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234244 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234222.285227:0:88982:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761234222.285228:0:88982:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234244 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234222.285232:0:88982:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234222.285233:0:88982:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:0.0:1761234222.285235:0:88982:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:0.0:1761234222.285237:0:88982:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009b62b3c1 (tot 1676282). 00000400:00000200:0.0:1761234222.285239:0:88982:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1761234222.285241:0:88982:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1761234222.285242:0:88982:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000003f47b3af 00000400:00000010:0.0:1761234222.285243:0:88982:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000003f47b3af. 00000100:00000001:0.0:1761234222.285244:0:88982:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:0.0:1761234222.285247:0:88982:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234244 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234222.285250:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1761234222.285252:0:88982:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:0.0:1761234222.285254:0:88982:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234244 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234222.285255:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234222.285259:0:88982:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234222.285259:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1761234222.285261:0:88982:0:(events.c:93:request_out_callback()) Process leaving 00000100:00100000:2.0:1761234222.285269:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234219/real 1761234222] req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234222.285289:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234222.285291:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234222.285294:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000ccddd6a4. 00000400:00000200:2.0:1761234222.285298:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000001d64cf84 00000400:00000010:2.0:1761234222.285299:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000001d64cf84. 00000100:00000001:2.0:1761234222.285302:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234222.285306:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234222.285321:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234222.285328:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234222.285330:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285333:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234222.285334:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234222.285338:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234222.285344:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234222.285348:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234222.285355:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234222.285358:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285359:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234222.285360:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285362:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234222.285364:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234222.285368:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234222.285371:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234222.285373:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234222.285376:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234222.285383:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233344:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234222.285387:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234222.285390:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234219/real 1761234222] req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234222.285396:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234222.285398:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234222.285400:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000003966d2af. 00000400:00000200:2.0:1761234222.285402:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234222.285403:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234222.285405:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234222.285406:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234222.285412:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234222.285417:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234222.285418:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285420:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234222.285421:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234222.285423:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234222.285428:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234222.285431:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234222.285436:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234222.285438:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285439:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234222.285440:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285442:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234222.285443:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234222.285445:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234222.285448:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234222.285450:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234222.285453:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234222.285460:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233408:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234222.285466:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234222.285469:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234219/real 1761234222] req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234222.285476:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234222.285477:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234222.285479:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234222.285480:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234222.285482:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234222.285484:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234222.285485:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234222.285493:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234222.285499:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234222.285500:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285502:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234222.285503:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234222.285505:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234222.285511:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234222.285514:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234222.285520:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234222.285522:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285523:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234222.285524:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285526:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234222.285527:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234222.285529:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234222.285532:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234222.285533:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234222.285537:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234222.285544:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233280:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234222.285548:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285551:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234222.285553:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002431276f x1846784245233344/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234222.285560:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234222.285563:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234222.285567:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234222.285572:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234222.285574:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234222.285576:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234222.285578:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234222.285581:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000009baf651a. 02000000:00000001:2.0:1761234222.285584:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234222.285586:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234222.285587:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002431276f. 00000100:00000001:2.0:1761234222.285591:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234222.285592:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285594:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234222.285596:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f61de4c8 x1846784245233408/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234222.285602:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234222.285603:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234222.285604:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000116ca75c. 02000000:00000001:2.0:1761234222.285607:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234222.285608:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234222.285609:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234222.285611:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234222.285613:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000df51b12a. 02000000:00000001:2.0:1761234222.285616:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234222.285617:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234222.285618:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f61de4c8. 00000100:00000001:2.0:1761234222.285622:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234222.285623:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285625:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234222.285635:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000108ab0a5 x1846784245233280/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234244 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234222.285642:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234222.285643:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234222.285644:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234222.285647:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234222.285648:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234222.285649:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234222.285652:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234222.285653:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000071d5cb10. 02000000:00000001:2.0:1761234222.285655:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234222.285656:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234222.285657:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000108ab0a5. 00000100:00000001:2.0:1761234222.285660:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234222.285661:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285663:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234222.285672:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234222.285674:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234222.285676:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234222.285677:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234222.796955:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234222.796960:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234222.796962:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234222.796967:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234222.796972:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234222.796976:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234222.796979:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234222.796981:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234222.796984:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234223.180961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234223.180965:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234223.180972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234223.180973:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234223.180976:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234223.180977:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234223.181095:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234223.181101:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234223.181105:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234223.820925:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234223.820930:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234223.820932:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234223.820935:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234223.820939:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234223.820942:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234223.820944:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234223.820946:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234223.820948:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234224.204971:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234224.204976:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234224.204983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234224.204984:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234224.204988:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234224.204989:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234224.205102:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234224.205105:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234224.205107:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234224.332973:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234224.332983:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234224.332984:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234224.332986:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234224.332988:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234224.332992:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234224.332998:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6693 00000100:00080000:1.0:1761234224.333000:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234224.333002:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333004:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234224.333006:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333007:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234224.333010:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234224.333011:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234224.333013:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234224.333015:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234224.333017:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234224.333018:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234224.333020:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333021:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234224.333022:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333023:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234224.333025:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234224.333027:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333028:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234224.333032:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234224.333035:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234224.333037:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333038:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234224.333039:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234224.333044:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000005940a194. 00000020:00000040:1.0:1761234224.333047:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234224.333050:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234224.333051:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333053:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234224.333056:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234224.333059:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234224.333065:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761234224.333070:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234224.333075:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234224.333082:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234224.333083:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234224.333086:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000005940a194] to pc [ptlrpcd_rcv+-1] req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234224.333102:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333104:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234224.333105:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234224.333107:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234224.333108:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234224.333108:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234224.333109:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234224.333111:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234224.333112:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6693 00000100:00080000:1.0:1761234224.333114:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234224.333115:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333116:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234224.333117:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333119:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234224.333120:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234224.333120:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234224.333121:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234224.333122:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234224.333124:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234224.333125:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234224.333127:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333128:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234224.333130:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333131:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234224.333133:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234224.333135:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333137:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234224.333140:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234224.333145:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234224.333147:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333149:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234224.333152:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234224.333155:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333157:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234224.333160:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007d6a5077. 00000020:00000040:1.0:1761234224.333163:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:2.0:1761234224.333163:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1761234224.333167:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:2.0:1761234224.333168:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1761234224.333169:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333172:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234224.333173:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234224.333174:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234224.333178:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234224.333181:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:1.0:1761234224.333182:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761234224.333188:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.333195:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1761234224.333196:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333198:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234224.333204:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234224.333207:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234224.333215:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000005940a194 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233472:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761234224.333218:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234224.333220:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.333222:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1761234224.333226:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007d6a5077] to pc [ptlrpcd_rcv+-1] req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234224.333228:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234224.333230:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234224.333235:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1761234224.333238:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234224.333241:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062f3eded. 00000100:00000001:1.0:1761234224.333242:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234224.333244:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1761234224.333246:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333252:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234224.333253:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000400:00000010:2.0:1761234224.333254:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00000001:1.0:1761234224.333256:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234224.333257:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1761234224.333264:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000001:1.0:1761234224.333267:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234224.333271:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6693 00000100:00000200:2.0:1761234224.333273:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233472, portal 25 00000100:00080000:1.0:1761234224.333275:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1761234224.333278:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1761234224.333280:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234224.333282:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039820992 : -104717669730624 : ffffa0c2847e76c0) 00000100:00000001:1.0:1761234224.333284:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234224.333289:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000040:2.0:1761234224.333293:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234249 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234224.333294:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234224.333299:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234224.333300:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234224.333302:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234224.333304:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000001:2.0:1761234224.333306:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000001:1.0:1761234224.333308:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234224.333312:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234224.333313:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234224.333318:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000040:1.0:1761234224.333319:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000200:2.0:1761234224.333322:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245233472, offset 0 00000100:00000001:1.0:1761234224.333323:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234224.333326:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333329:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234224.333331:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000200:2.0:1761234224.333336:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00080000:1.0:1761234224.333337:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234224.333341:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234224.333343:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234224.333346:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234224.333348:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333350:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234224.333352:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1761234224.333354:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000008:00000001:1.0:1761234224.333356:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333359:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234224.333364:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000643267a. 00000400:00000200:2.0:1761234224.333366:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000020:00000040:1.0:1761234224.333367:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234224.333372:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234224.333374:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234224.333374:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1761234224.333377:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234224.333380:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1761234224.333380:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 02000000:00000001:1.0:1761234224.333384:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234224.333388:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11865:11865:256:4294967295] 192.168.201.124@tcp LPNI seq info [11865:11865:8:4294967295] 02000000:00000010:1.0:1761234224.333390:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000001:1.0:1761234224.333395:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234224.333401:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234224.333411:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:1.0:1761234224.333417:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234224.333419:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1761234224.333423:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000040:1.0:1761234224.333425:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000643267a] to pc [ptlrpcd_rcv+-1] req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000800:00000010:2.0:1761234224.333432:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000100:00000001:1.0:1761234224.333437:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234224.333441:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000200:2.0:1761234224.333444:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000040:1.0:1761234224.333445:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6703) 00000800:00000200:2.0:1761234224.333450:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761234224.333459:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234224.333462:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1761234224.333466:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234224.333469:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234224.333474:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234224.333479:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234224.333484:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234224.333486:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234224.333491:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234224.333494:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234249 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333503:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234224.333506:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234249 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333512:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.333514:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234224.333520:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.333522:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234224.333525:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.333528:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234224.333531:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234224/real 1761234224] req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234224.333540:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234224.333542:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234224.333544:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234224.333546:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234224.333548:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234224.333550:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234224.333552:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234224.333559:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333566:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234224.333568:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.333570:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234224.333572:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234224.333575:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333581:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234224.333585:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234224.333592:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234224.333595:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.333597:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234224.333598:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.333600:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234224.333602:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234224.333604:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234224.333608:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234224.333610:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234224.333613:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234224.333620:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000005940a194 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233472:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234224.333625:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.333627:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234224.333629:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000005940a194 x1846784245233472/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234224.333636:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234224.333639:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234224.333641:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234224.333645:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234224.333647:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234224.333648:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234224.333650:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234224.333652:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006d8d01a9. 02000000:00000001:2.0:1761234224.333656:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234224.333657:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234224.333659:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000005940a194. 00000100:00000001:2.0:1761234224.333662:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234224.333663:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.333665:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.333673:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234224.333675:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.333677:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234224.333678:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234224.333680:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234224.333682:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333689:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234224.333690:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234224.333692:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234224.333693:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234224.333695:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007d6a5077 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233536:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234224.333698:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234224.333700:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234224.333701:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234224.333703:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234224.333705:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234224.333707:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234224.333710:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234224.333712:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234224.333715:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233536, portal 10 00000100:00000001:2.0:1761234224.333716:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234224.333717:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039794240 : -104717669757376 : ffffa0c2847e0e40) 00000100:00000040:2.0:1761234224.333723:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234249 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333734:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234224.333737:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234224.333738:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234224.333741:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245233536, offset 0 00000400:00000200:2.0:1761234224.333745:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234224.333751:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234224.333754:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234224.333756:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234224.333798:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234224.333801:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11866:11866:256:4294967295] 192.168.201.124@tcp LPNI seq info [11866:11866:8:4294967295] 00000400:00000200:2.0:1761234224.333809:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234224.333814:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234224.333817:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234224.333821:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234224.333823:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761234224.333827:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234224.333829:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1761234224.333832:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234224.333834:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234224.333836:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234224.333840:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234224.333842:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234224.333871:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234224.333875:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234224.333879:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234249 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333887:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234224.333889:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234249 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333895:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.333896:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234224.333901:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.333902:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234224.333904:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.333906:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234224.333908:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234224/real 1761234224] req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234224.333916:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234224.333917:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234224.333918:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234224.333920:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234224.333922:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234224.333923:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234224.333925:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234224.333933:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333939:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234224.333940:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.333942:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234224.333943:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234224.333945:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333952:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234224.333954:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234224.333961:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234224.333963:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.333965:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234224.333966:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.333968:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234224.333969:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234224.333971:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234224.333974:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234224.333976:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234224.333980:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234224.333987:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007d6a5077 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233536:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234224.333991:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234224.333993:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.333998:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234224.333999:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234224.334000:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234224.334002:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234224.334004:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000643267a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233600:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234224.334007:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234224.334009:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234224.334011:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234224.334013:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234224.334032:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000116ca75c. 02000000:00000001:2.0:1761234224.334035:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234224.334038:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234224.334042:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234224.334044:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233600, portal 4 00000100:00000001:2.0:1761234224.334046:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234224.334047:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039800320 : -104717669751296 : ffffa0c2847e2600) 00000100:00000040:2.0:1761234224.334052:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234249 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.334057:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234224.334060:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234224.334062:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234224.334065:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245233600, offset 0 00000400:00000200:2.0:1761234224.334069:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234224.334075:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234224.334078:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234224.334081:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234224.334083:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234224.334085:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11867:11867:256:4294967295] 192.168.201.124@tcp LPNI seq info [11867:11867:8:4294967295] 00000400:00000200:2.0:1761234224.334092:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234224.334096:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234224.334100:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234224.334103:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234224.334105:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1761234224.334108:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234224.334110:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1761234224.334114:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234224.334116:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234224.334119:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234224.334122:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234224.334125:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234224.334126:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234224.334128:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234224.334131:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234249 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.334136:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234224.334138:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234249 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.334143:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.334144:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234224.334147:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.334148:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234224.334150:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.334152:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234224.334154:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234224/real 1761234224] req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234224.334161:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234224.334163:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234224.334164:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234224.334166:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234224.334167:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234224.334169:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234224.334170:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234224.334177:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.334183:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234224.334184:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.334185:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234224.334186:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234224.334188:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234224.334194:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234224.334196:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234224.334202:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234224.334203:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.334204:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234224.334205:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.334207:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234224.334208:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234224.334210:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234224.334212:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234224.334214:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234224.334217:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234224.334222:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000643267a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233600:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234224.334236:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.334238:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234224.334239:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007d6a5077 x1846784245233536/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234224.334244:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234224.334246:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234224.334247:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234224.334250:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234224.334251:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234224.334252:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234224.334254:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234224.334255:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:2.0:1761234224.334258:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234224.334259:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234224.334259:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007d6a5077. 00000100:00000001:2.0:1761234224.334262:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234224.334263:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.334264:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234224.334266:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000643267a x1846784245233600/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234249 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234224.334271:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234224.334272:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234224.334273:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000116ca75c. 02000000:00000001:2.0:1761234224.334275:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234224.334276:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234224.334276:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234224.334278:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234224.334279:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:2.0:1761234224.334281:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234224.334282:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234224.334283:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000643267a. 00000100:00000001:2.0:1761234224.334285:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234224.334286:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.334287:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234224.334291:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234224.334292:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234224.334294:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234224.334295:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234224.844961:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234224.844966:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234224.844967:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234224.844971:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234224.844975:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234224.844978:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234224.844979:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234224.844981:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234224.844983:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234225.228977:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234225.228984:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234225.228994:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234225.228995:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234225.228999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234225.229000:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234225.229082:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234225.229087:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234225.229090:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234225.868917:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234225.868923:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234225.868925:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234225.868930:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234225.868935:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234225.868938:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234225.868940:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234225.868942:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234225.868944:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234226.252982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234226.252990:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234226.252999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234226.253000:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234226.253004:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234226.253005:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234226.253108:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234226.253112:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234226.253116:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234226.892982:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234226.892987:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234226.892989:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234226.892994:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234226.893000:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234226.893004:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234226.893007:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234226.893009:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234226.893012:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234227.276948:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234227.276953:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234227.276958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234227.276960:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234227.276963:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234227.276964:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234227.277070:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234227.277073:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234227.277076:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234227.916935:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234227.916939:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234227.916940:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234227.916945:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234227.916949:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234227.916951:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234227.916953:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234227.916955:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234227.916957:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234228.301007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234228.301015:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234228.301025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234228.301026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234228.301030:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234228.301031:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234228.301100:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234228.301103:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234228.301106:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234228.940964:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234228.940968:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234228.940970:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234228.940973:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234228.940976:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234228.940979:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234228.940981:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234228.940982:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234228.940984:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234229.324952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234229.324957:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234229.324965:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234229.324966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234229.324970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234229.324972:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234229.325042:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234229.325045:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234229.325048:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234229.452980:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234229.452992:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234229.452994:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234229.452997:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234229.452999:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234229.453004:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234229.453012:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6698 00000100:00080000:1.0:1761234229.453015:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234229.453018:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453021:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234229.453023:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453025:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234229.453029:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234229.453031:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234229.453033:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234229.453036:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234229.453038:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234229.453040:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234229.453043:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453045:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234229.453046:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453048:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234229.453050:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234229.453054:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453057:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234229.453060:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234229.453064:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234229.453066:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234229.453069:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234229.453071:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234229.453079:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000fd54b549. 00000020:00000040:1.0:1761234229.453082:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234229.453086:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234229.453087:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234229.453090:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234229.453093:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234229.453096:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234229.453103:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a3ed3058. 00000100:00000001:1.0:1761234229.453110:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234229.453117:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234229.453126:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234229.453129:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234229.453133:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000fd54b549] to pc [ptlrpcd_rcv+-1] req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234229.453157:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234229.453159:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234229.453161:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234229.453164:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234229.453165:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234229.453167:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234229.453168:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234229.453170:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234229.453172:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6698 00000100:00080000:1.0:1761234229.453174:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234229.453177:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453178:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234229.453180:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453181:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234229.453184:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234229.453185:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234229.453186:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234229.453188:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234229.453190:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234229.453192:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234229.453195:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453198:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234229.453200:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453203:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234229.453206:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00000001:2.0:1761234229.453209:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1761234229.453210:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453214:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453214:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1761234229.453218:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1761234229.453218:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1761234229.453224:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1761234229.453224:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:1.0:1761234229.453229:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234229.453233:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234229.453236:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234229.453242:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453244:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234229.453246:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453246:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234229.453252:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 02000000:00000001:2.0:1761234229.453253:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234229.453255:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1761234229.453257:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00100000:2.0:1761234229.453259:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233664:192.168.201.124@tcp:250: 02000000:00000001:1.0:1761234229.453263:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:2.0:1761234229.453264:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1761234229.453266:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234229.453267:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1761234229.453270:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234229.453270:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234229.453273:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234229.453274:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1761234229.453277:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234229.453279:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234229.453282:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234229.453285:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000400:00000010:2.0:1761234229.453287:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00000001:1.0:1761234229.453291:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234229.453294:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00100000:1.0:1761234229.453299:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234229.453300:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233664, portal 25 00000100:00000001:2.0:1761234229.453304:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234229.453306:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039795456 : -104717669756160 : ffffa0c2847e1300) 00000100:00000001:1.0:1761234229.453313:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1761234229.453313:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234254 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234229.453315:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453322:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:1.0:1761234229.453323:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234229.453327:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234229.453330:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234229.453333:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245233664, offset 0 00000400:00000200:2.0:1761234229.453340:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453341:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234229.453345:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234229.453346:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234229.453350:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234229.453352:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234229.453354:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1761234229.453354:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00080000:1.0:1761234229.453357:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234229.453360:0:468:0:(import.c:519:import_select_connection()) Process entered 00000400:00000200:2.0:1761234229.453361:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00080000:1.0:1761234229.453365:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6698 00000400:00000200:2.0:1761234229.453366:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1761234229.453370:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1761234229.453371:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000040:1.0:1761234229.453378:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000200:2.0:1761234229.453379:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11868:11868:256:4294967295] 192.168.201.124@tcp LPNI seq info [11868:11868:8:4294967295] 00000100:00000001:1.0:1761234229.453383:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234229.453388:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453393:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000200:2.0:1761234229.453393:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000020:00000001:1.0:1761234229.453398:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234229.453400:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000800:00000200:2.0:1761234229.453400:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000020:00000001:1.0:1761234229.453405:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000800:00000010:2.0:1761234229.453406:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000020:00000040:1.0:1761234229.453408:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234229.453412:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000800:00000200:2.0:1761234229.453414:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000020:00000001:1.0:1761234229.453416:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000800:00000200:2.0:1761234229.453418:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1761234229.453424:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000800:00000200:2.1:1761234229.453426:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:1.0:1761234229.453429:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1:1761234229.453430:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000100:00000040:1.0:1761234229.453434:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000800:00000100:2.0:1761234229.453438:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234229.453439:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000800:00000010:2.0:1761234229.453441:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000020:00000040:1.0:1761234229.453444:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000200:2.0:1761234229.453446:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:1.0:1761234229.453449:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000200:2.0:1761234229.453454:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1761234229.453455:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234229.453458:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1761234229.453458:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234229.453461:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 02000000:00000001:1.0:1761234229.453463:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234229.453466:0:88993:0:(events.c:53:request_out_callback()) Process entered 02000000:00000001:1.0:1761234229.453468:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234229.453469:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234254 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234229.453472:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234229.453474:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1761234229.453477:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000008:00000001:1.0:1761234229.453478:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234229.453481:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234254 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234229.453483:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234229.453489:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000100:00000001:2.0:1761234229.453489:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453491:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000020:00000040:1.0:1761234229.453494:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000001:2.0:1761234229.453495:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234229.453498:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:2.0:1761234229.453499:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234229.453501:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453502:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234229.453506:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234229.453507:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1761234229.453510:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1761234229.453510:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234229/real 1761234229] req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 02000000:00000001:1.0:1761234229.453512:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234229.453518:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000400:00000001:2.0:1761234229.453519:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234229.453522:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1761234229.453523:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234229.453524:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234229.453526:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234229.453528:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234229.453530:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00100000:1.0:1761234229.453531:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234229.453534:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234229.453559:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1761234229.453560:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234229.453562:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453568:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234229.453570:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761234229.453571:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453573:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234229.453574:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234229.453581:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234229.453587:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453589:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1761234229.453591:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:2.0:1761234229.453594:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000040:1.0:1761234229.453596:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6708) 00000100:00000001:2.0:1761234229.453601:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234229.453603:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453604:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234229.453605:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453607:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234229.453608:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234229.453609:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234229.453611:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234229.453612:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234229.453615:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234229.453620:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000fd54b549 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233664:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234229.453623:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453625:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234229.453627:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000fd54b549 x1846784245233664/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234229.453631:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234229.453633:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234229.453634:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234229.453636:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234229.453637:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234229.453638:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234229.453639:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234229.453641:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a3ed3058. 02000000:00000001:2.0:1761234229.453644:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234229.453645:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234229.453646:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000fd54b549. 00000100:00000001:2.0:1761234229.453648:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234229.453649:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453650:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453655:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234229.453656:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453658:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234229.453658:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234229.453659:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234229.453661:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453693:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234229.453694:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234229.453695:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234229.453696:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234229.453698:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233728:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234229.453700:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234229.453701:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234229.453702:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234229.453703:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234229.453705:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234229.453707:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234229.453708:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234229.453710:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234229.453712:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233728, portal 10 00000100:00000001:2.0:1761234229.453713:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234229.453714:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039808832 : -104717669742784 : ffffa0c2847e4740) 00000100:00000040:2.0:1761234229.453718:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234254 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453722:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234229.453724:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234229.453725:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234229.453727:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245233728, offset 0 00000400:00000200:2.0:1761234229.453730:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234229.453733:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234229.453736:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234229.453737:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234229.453739:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234229.453741:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11869:11869:256:4294967295] 192.168.201.124@tcp LPNI seq info [11869:11869:8:4294967295] 00000400:00000200:2.0:1761234229.453746:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234229.453749:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234229.453750:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234229.453753:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234229.453754:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761234229.453757:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234229.453758:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1761234229.453760:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234229.453762:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234229.453763:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234229.453773:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234229.453775:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234229.453775:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234229.453777:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234229.453778:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234254 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453783:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234229.453784:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234254 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453788:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453789:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234229.453791:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453792:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234229.453793:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453794:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234229.453796:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234229/real 1761234229] req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234229.453801:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234229.453802:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234229.453802:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234229.453803:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234229.453804:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234229.453805:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234229.453806:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234229.453810:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453817:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234229.453818:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453819:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234229.453819:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234229.453821:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453824:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234229.453826:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234229.453829:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234229.453831:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453831:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234229.453832:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453833:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234229.453834:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234229.453836:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234229.453837:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234229.453838:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234229.453840:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234229.453869:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233728:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234229.453871:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234229.453873:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453876:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234229.453877:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234229.453878:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234229.453880:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234229.453881:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233792:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234229.453883:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234229.453884:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234229.453885:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234229.453886:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234229.453888:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234229.453889:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234229.453891:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234229.453892:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234229.453894:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233792, portal 4 00000100:00000001:2.0:1761234229.453895:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234229.453896:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039812480 : -104717669739136 : ffffa0c2847e5580) 00000100:00000040:2.0:1761234229.453898:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234254 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453902:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234229.453904:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234229.453905:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234229.453906:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245233792, offset 0 00000400:00000200:2.0:1761234229.453908:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234229.453911:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234229.453913:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234229.453915:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234229.453916:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234229.453918:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11870:11870:256:4294967295] 192.168.201.124@tcp LPNI seq info [11870:11870:8:4294967295] 00000400:00000200:2.0:1761234229.453923:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234229.453925:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234229.453927:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234229.453929:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234229.453931:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000200:2.1:1761234229.453933:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234229.453934:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 53 secs later) 00000800:00000100:2.0:1761234229.453937:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234229.453938:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234229.453940:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234229.453941:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234229.453943:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234229.453944:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234229.453945:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234229.453947:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234254 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453951:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234229.453952:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234254 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453956:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453956:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234229.453958:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453959:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234229.453960:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.453961:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234229.453962:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234229/real 1761234229] req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234229.453967:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234229.453967:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234229.453968:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234229.453969:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234229.453970:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234229.453971:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234229.453972:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234229.453977:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453980:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234229.453980:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453981:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234229.453982:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234229.453983:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234229.453987:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234229.453988:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234229.453991:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234229.453993:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453993:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234229.453994:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.453995:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234229.453995:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234229.453997:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234229.453998:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234229.453999:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234229.454001:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234229.454005:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233792:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234229.454007:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.454008:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234229.454009:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f33bf85c x1846784245233728/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234229.454013:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234229.454014:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234229.454015:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234229.454016:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234229.454017:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234229.454018:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234229.454019:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234229.454019:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:2.0:1761234229.454021:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234229.454022:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234229.454022:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f33bf85c. 00000100:00000001:2.0:1761234229.454024:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234229.454025:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.454026:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234229.454027:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004567def3 x1846784245233792/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234254 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234229.454030:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234229.454030:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234229.454031:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234229.454032:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234229.454033:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234229.454033:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234229.454034:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234229.454035:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:2.0:1761234229.454036:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234229.454037:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234229.454037:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004567def3. 00000100:00000001:2.0:1761234229.454039:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234229.454039:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.454040:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234229.454042:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234229.454043:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234229.454044:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234229.454045:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234229.964936:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234229.964943:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234229.964945:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234229.964950:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234229.964956:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234229.964959:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234229.964970:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234229.964972:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234229.964975:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234230.348952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234230.348956:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234230.348961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234230.348962:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234230.348965:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234230.348966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234230.349060:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234230.349063:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234230.349066:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1761234230.988928:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1761234230.988932:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234230.988933:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234230.988947:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:0.0:1761234230.988955:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234230.988958:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234230.988960:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234230.988962:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234230.988965:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.372968:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234231.372973:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234231.372975:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234231.372977:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.372981:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234231.372983:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.372985:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.372986:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.372989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234231.372990:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234231.372990:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234231.372994:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234231.372996:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234231.372997:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.372998:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234231.372998:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.372999:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.373000:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.373002:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234231.373003:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234231.373004:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234231.373005:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234231.373006:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234231.373006:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.373007:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234231.373008:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.373008:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234231.373009:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234231.373080:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234231.373084:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234231.373087:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234232.012942:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234232.012947:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234232.012950:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234232.012955:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234232.012960:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234232.012962:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234232.012965:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234232.012968:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234232.012970:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234232.396945:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234232.396950:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234232.396955:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234232.396956:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234232.396958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234232.396959:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234232.397035:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234232.397038:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234232.397041:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234233.036937:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234233.036944:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234233.036946:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234233.036951:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234233.036956:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234233.036969:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234233.036972:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234233.036975:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234233.036977:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234233.420922:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234233.420926:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234233.420931:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234233.420931:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234233.420934:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234233.420934:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234233.421023:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234233.421028:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234233.421033:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234234.060955:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234234.060961:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234234.060962:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234234.060967:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234234.060972:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234234.060975:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.060977:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234234.060979:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.060982:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234234.444957:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234234.444961:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234234.444966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234234.444968:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234234.444970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234234.444972:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234234.445040:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234234.445043:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234234.445046:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234234.572933:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234234.572944:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234234.572946:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234234.572949:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234234.572951:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234234.572956:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234234.572963:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6703 00000100:00080000:1.0:1761234234.572966:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234234.572969:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234234.572971:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234234.572973:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234234.572975:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234234.572979:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234234.572981:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234234.572984:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234234.572986:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234234.572989:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234234.572991:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234234.572995:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234234.572996:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234234.572998:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234234.573000:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234234.573002:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234234.573005:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234234.573007:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234234.573011:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234234.573014:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234234.573017:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234234.573019:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234234.573021:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234234.573030:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761234234.573034:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234234.573038:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234234.573039:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234234.573042:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234234.573045:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234234.573048:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234234.573055:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f654abd8. 00000100:00000001:1.0:1761234234.573062:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234234.573069:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234234.573078:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234234.573080:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234234.573084:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234234.573109:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234234.573111:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234234.573113:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234234.573116:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234234.573117:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234234.573118:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234234.573120:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234234.573122:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234234.573125:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6703 00000100:00080000:1.0:1761234234.573127:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234234.573129:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234234.573131:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234234.573133:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234234.573134:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234234.573137:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234234.573138:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234234.573140:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234234.573142:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234234.573144:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234234.573146:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234234.573150:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234234.573153:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234234.573154:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234234.573156:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1761234234.573157:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234234.573160:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00000001:2.0:1761234234.573160:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234234.573163:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1761234234.573165:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000040:2.0:1761234234.573169:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234234.573170:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234234.573173:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1761234234.573178:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1761234234.573179:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234234.573180:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234234.573185:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234234.573185:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1761234234.573189:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 02000000:00000001:2.0:1761234234.573189:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234234.573192:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233856:192.168.201.124@tcp:250: 00000008:00100000:1.0:1761234234.573193:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1761234234.573197:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000008:00000001:1.0:1761234234.573199:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234234.573201:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1761234234.573202:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234234.573207:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761234234.573210:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:2.0:1761234234.573212:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234234.573213:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:2.0:1761234234.573215:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1761234234.573216:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234234.573220:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:2.0:1761234234.573220:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:1.0:1761234234.573221:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234234.573224:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234234.573226:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234234.573229:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 02000000:00000010:1.0:1761234234.573232:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000400:00000010:2.0:1761234234.573236:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000001:1.0:1761234234.573238:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234234.573243:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233856, portal 25 00000100:00100000:1.0:1761234234.573246:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573247:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234234.573248:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039807616 : -104717669744000 : ffffa0c2847e4280) 00000100:00000001:1.0:1761234234.573257:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1761234234.573258:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234259 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234234.573260:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234234.573268:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573269:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234234.573275:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234234.573277:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234234.573281:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245233856, offset 0 00000100:00000001:1.0:1761234234.573286:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234234.573289:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234234.573291:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234234.573293:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234234.573297:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234234.573298:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234234.573299:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234234.573301:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1761234234.573304:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:1.0:1761234234.573306:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234234.573310:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6703 00000400:00000200:2.0:1761234234.573313:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00080000:1.0:1761234234.573314:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1761234234.573318:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1761234234.573320:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234234.573324:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1761234234.573324:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000040:1.0:1761234234.573330:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000400:00000200:2.0:1761234234.573331:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11871:11871:256:4294967295] 192.168.201.124@tcp LPNI seq info [11871:11871:8:4294967295] 00000100:00000001:1.0:1761234234.573335:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234234.573339:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234234.573341:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234234.573344:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000400:00000200:2.0:1761234234.573344:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000020:00000040:1.0:1761234234.573346:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234234.573350:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000800:00000200:2.0:1761234234.573351:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000020:00000001:1.0:1761234234.573356:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000800:00000010:2.0:1761234234.573356:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000100:00000040:1.0:1761234234.573363:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000800:00000200:2.0:1761234234.573365:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:1.0:1761234234.573368:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1761234234.573369:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000100:00000040:1.0:1761234234.573373:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000800:00000200:2.1:1761234234.573378:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:1.0:1761234234.573379:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000800:00000200:2.1:1761234234.573381:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000020:00000040:1.0:1761234234.573383:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000800:00000100:2.0:1761234234.573387:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000100:00080000:1.0:1761234234.573390:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000800:00000010:2.0:1761234234.573390:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000100:00000001:1.0:1761234234.573395:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234234.573397:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1761234234.573398:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234234.573402:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1761234234.573402:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234234.573405:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 02000000:00000001:1.0:1761234234.573407:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234234.573408:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:1.0:1761234234.573411:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234234.573413:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000008:00100000:1.0:1761234234.573414:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1761234234.573418:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234259 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000008:00000001:1.0:1761234234.573419:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234234.573423:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234234.573428:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000100:00000001:2.0:1761234234.573429:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234234.573432:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234259 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000020:00000040:1.0:1761234234.573433:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:1.0:1761234234.573438:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234234.573439:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573440:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234234.573443:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234234.573443:0:88993:0:(events.c:93:request_out_callback()) Process leaving 02000000:00000001:1.0:1761234234.573446:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234234.573449:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573449:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573452:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1761234234.573454:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:2.0:1761234234.573454:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573459:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1761234234.573460:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234234.573461:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234234/real 1761234234] req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761234234.573468:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234234.573471:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234234.573472:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234234.573473:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234234.573475:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234234.573477:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:1.0:1761234234.573480:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234234.573480:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000001:1.0:1761234234.573482:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234234.573486:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:1.0:1761234234.573490:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234234.573499:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234234.573510:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573511:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:1.0:1761234234.573513:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000001:2.0:1761234234.573514:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.573517:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234234.573518:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1761234234.573519:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6713) 00000100:00100000:2.0:1761234234.573522:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573527:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234234.573529:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234234.573533:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234234.573535:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.573536:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234234.573537:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.573539:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234234.573540:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234234.573542:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234234.573544:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234234.573545:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234234.573547:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234234.573552:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245233856:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234234.573555:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.573556:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234234.573557:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a014c997 x1846784245233856/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234234.573561:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234234.573563:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234234.573577:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234234.573580:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234234.573581:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234234.573582:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234234.573584:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234234.573585:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f654abd8. 02000000:00000001:2.0:1761234234.573588:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234234.573589:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234234.573590:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a014c997. 00000100:00000001:2.0:1761234234.573592:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234234.573593:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.573594:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.573600:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234234.573601:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573602:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234234.573603:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234234.573604:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234234.573607:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573613:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234234.573615:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234234.573616:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234234.573618:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234234.573620:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002b682c08 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233920:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234234.573623:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234234.573625:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234234.573626:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234234.573628:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234234.573630:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234234.573632:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234234.573635:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234234.573638:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234234.573640:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233920, portal 10 00000100:00000001:2.0:1761234234.573642:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234234.573643:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039801536 : -104717669750080 : ffffa0c2847e2ac0) 00000100:00000040:2.0:1761234234.573649:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234259 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573655:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234234.573658:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234234.573659:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234234.573661:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245233920, offset 0 00000400:00000200:2.0:1761234234.573665:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234234.573670:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234234.573674:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234234.573676:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234234.573678:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234234.573681:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11872:11872:256:4294967295] 192.168.201.124@tcp LPNI seq info [11872:11872:8:4294967295] 00000400:00000200:2.0:1761234234.573688:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234234.573692:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234234.573695:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234234.573699:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234234.573701:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761234234.573705:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234234.573707:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1761234234.573710:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234234.573712:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234234.573714:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234234.573718:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234234.573720:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234234.573722:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234234.573724:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234234.573726:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234259 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573733:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234234.573735:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234259 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573741:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573742:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234234.573745:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573747:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234234.573748:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573750:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234234.573753:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234234/real 1761234234] req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234234.573760:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234234.573761:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234234.573763:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234234.573765:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234234.573766:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234234.573767:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234234.573769:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234234.573776:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573782:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234234.573783:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.573785:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234234.573786:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234234.573788:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573794:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234234.573797:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234234.573802:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234234.573804:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.573805:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234234.573806:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.573808:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234234.573809:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234234.573811:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234234.573814:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234234.573816:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234234.573819:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234234.573826:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002b682c08 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233920:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234234.573830:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234234.573832:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573838:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234234.573839:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234234.573841:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234234.573842:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234234.573872:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233984:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234234.573876:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234234.573877:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234234.573879:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234234.573880:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234234.573883:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234234.573884:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234234.573887:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234234.573890:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234234.573893:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245233984, portal 4 00000100:00000001:2.0:1761234234.573894:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234234.573895:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039814912 : -104717669736704 : ffffa0c2847e5f00) 00000100:00000040:2.0:1761234234.573900:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234259 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573906:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234234.573908:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234234.573910:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234234.573911:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245233984, offset 0 00000400:00000200:2.0:1761234234.573915:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234234.573920:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234234.573923:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234234.573926:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234234.573928:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234234.573931:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11873:11873:256:4294967295] 192.168.201.124@tcp LPNI seq info [11873:11873:8:4294967295] 00000400:00000200:2.0:1761234234.573938:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234234.573943:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234234.573945:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234234.573949:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234234.573951:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000200:2.1:1761234234.573954:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234234.573956:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 48 secs later) 00000800:00000100:2.0:1761234234.573959:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234234.573961:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234234.573963:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234234.573966:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234234.573969:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234234.573970:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234234.573973:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234234.573975:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234259 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573982:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234234.573985:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234259 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.573991:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573992:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234234.573994:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573996:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234234.573998:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.573999:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234234.574002:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234234/real 1761234234] req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234234.574008:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234234.574010:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234234.574011:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234234.574013:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234234.574014:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234234.574016:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234234.574018:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234234.574025:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.574031:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234234.574032:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.574033:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234234.574035:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234234.574037:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234234.574043:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234234.574046:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234234.574052:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234234.574054:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.574056:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234234.574057:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.574058:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234234.574059:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234234.574062:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234234.574064:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234234.574066:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234234.574069:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234234.574076:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245233984:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234234.574082:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.574084:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234234.574086:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002b682c08 x1846784245233920/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234234.574090:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234234.574091:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234234.574092:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234234.574094:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234234.574095:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234234.574096:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234234.574097:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234234.574098:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:2.0:1761234234.574099:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234234.574100:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234234.574101:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002b682c08. 00000100:00000001:2.0:1761234234.574102:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234234.574103:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.574104:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234234.574105:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000632be758 x1846784245233984/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234259 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234234.574109:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234234.574109:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234234.574110:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000062f3eded. 02000000:00000001:2.0:1761234234.574111:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234234.574111:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234234.574112:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234234.574113:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234234.574113:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:2.0:1761234234.574114:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234234.574115:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234234.574115:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000632be758. 00000100:00000001:2.0:1761234234.574117:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234234.574117:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.574118:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234234.574121:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234234.574121:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234234.574123:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234234.574124:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234235.084996:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234235.085002:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234235.085004:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234235.085008:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234235.085013:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234235.085017:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234235.085019:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234235.085022:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234235.085024:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234235.468931:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234235.468935:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234235.468940:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234235.468941:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234235.468943:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234235.468944:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234235.469003:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234235.469006:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234235.469009:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234236.108956:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234236.108961:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234236.108962:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234236.108965:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234236.108969:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234236.108972:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234236.108974:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234236.108976:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234236.108978:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234236.492981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234236.492985:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234236.492991:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234236.492992:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234236.492994:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234236.492995:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234236.493085:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234236.493092:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234236.493097:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234237.133027:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234237.133034:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234237.133036:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234237.133042:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234237.133050:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234237.133054:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234237.133057:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234237.133060:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234237.133064:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234237.517011:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234237.517016:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234237.517023:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234237.517026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234237.517029:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234237.517031:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234237.517101:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234237.517104:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234237.517107:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234238.156957:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234238.156961:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234238.156962:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234238.156965:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234238.156970:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234238.156973:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234238.156974:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234238.156976:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234238.156977:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234238.540959:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234238.540965:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234238.540972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234238.540973:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234238.540976:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234238.540977:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234238.541035:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234238.541038:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234238.541041:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234239.180969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234239.180973:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234239.180974:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234239.180978:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234239.180982:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234239.180985:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234239.180986:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234239.180988:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234239.180990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234239.565000:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234239.565004:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234239.565009:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234239.565010:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234239.565013:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234239.565014:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234239.565088:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234239.565091:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234239.565093:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234239.692987:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234239.692996:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234239.692998:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234239.693000:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234239.693002:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234239.693006:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234239.693012:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6708 00000100:00080000:1.0:1761234239.693014:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234239.693016:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693018:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234239.693020:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693021:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234239.693024:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234239.693025:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234239.693028:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234239.693030:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234239.693031:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234239.693033:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234239.693035:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693036:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234239.693037:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693038:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234239.693040:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234239.693042:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693043:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234239.693045:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234239.693048:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234239.693049:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693051:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234239.693052:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234239.693061:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000bb0f086. 00000020:00000040:1.0:1761234239.693064:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234239.693067:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234239.693068:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693070:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234239.693072:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234239.693075:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234239.693081:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000a174ebf2. 00000100:00000001:1.0:1761234239.693086:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234239.693091:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234239.693098:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234239.693099:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234239.693102:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000bb0f086] to pc [ptlrpcd_rcv+-1] req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234239.693119:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693120:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234239.693122:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234239.693124:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234239.693124:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234239.693125:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234239.693126:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234239.693128:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234239.693129:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6708 00000100:00080000:1.0:1761234239.693130:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234239.693131:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693132:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234239.693133:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693134:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234239.693135:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234239.693136:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234239.693137:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234239.693138:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234239.693139:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234239.693141:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234239.693143:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693144:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234239.693145:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693146:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234239.693148:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234239.693149:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693151:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234239.693152:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234239.693156:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234239.693157:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693159:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234239.693161:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1761234239.693162:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000008:00000001:1.0:1761234239.693163:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693166:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693167:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000010:1.0:1761234239.693169:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000632be758. 00000100:00000001:2.0:1761234239.693170:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1761234239.693173:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:1.0:1761234239.693175:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234239.693176:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234239.693177:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234239.693178:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234239.693179:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234239.693180:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234239.693184:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000f43b35ab. 00000100:00000001:1.0:1761234239.693188:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693190:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234239.693192:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234239.693193:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234239.693196:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1761234239.693199:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1761234239.693200:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693202:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234239.693205:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234048:192.168.201.124@tcp:250: 00000100:00000040:1.0:1761234239.693207:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000632be758] to pc [ptlrpcd_rcv+-1] req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693213:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234239.693216:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234239.693218:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234239.693221:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1761234239.693223:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693225:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 02000000:00000010:2.0:1761234239.693226:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 00000100:00080000:1.0:1761234239.693227:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234239.693231:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 02000000:00000001:2.0:1761234239.693231:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1761234239.693234:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234239.693235:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234239.693236:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234239.693238:0:468:0:(import.c:519:import_select_connection()) Process entered 00000400:00000010:2.0:1761234239.693238:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00080000:1.0:1761234239.693241:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6708 00000100:00080000:1.0:1761234239.693243:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234239.693244:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000010:2.0:1761234239.693245:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000001:1.0:1761234239.693246:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234239.693248:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693250:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234239.693252:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000100:00000200:2.0:1761234239.693252:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234048, portal 25 00000020:00000040:1.0:1761234239.693253:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234239.693255:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234239.693257:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000001:2.0:1761234239.693257:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234239.693259:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039803968 : -104717669747648 : ffffa0c2847e3440) 00000020:00000001:1.0:1761234239.693260:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:1.0:1761234239.693264:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:1.0:1761234239.693268:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000040:2.0:1761234239.693268:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234264 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234239.693271:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234239.693274:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693277:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234239.693280:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234239.693281:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234239.693282:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234239.693283:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234239.693286:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234239.693289:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693292:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234239.693293:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234239.693296:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693298:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693301:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000010:1.0:1761234239.693302:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002b682c08. 00000020:00000040:1.0:1761234239.693306:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000040:2.0:1761234239.693306:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 02000000:00000001:1.0:1761234239.693308:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234239.693310:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234239.693311:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1761234239.693311:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 02000000:00000001:1.0:1761234239.693313:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234239.693315:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234239.693315:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245234048, offset 0 02000000:00000010:1.0:1761234239.693321:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d13d0f57. 00000100:00000001:1.0:1761234239.693324:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234239.693327:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00100000:1.0:1761234239.693330:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234239.693335:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234239.693336:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234239.693337:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002b682c08] to pc [ptlrpcd_rcv+-1] req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234239.693342:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234239.693342:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:1.0:1761234239.693344:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234239.693348:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6718) 00000400:00000200:2.0:1761234239.693351:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234239.693355:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234239.693358:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234239.693360:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11874:11874:256:4294967295] 192.168.201.124@tcp LPNI seq info [11874:11874:8:4294967295] 00000400:00000200:2.0:1761234239.693371:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234239.693376:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234239.693380:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234239.693385:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234239.693387:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761234239.693392:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234239.693394:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1761234239.693397:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234239.693399:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234239.693403:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234239.693406:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234239.693410:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234239.693411:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234239.693415:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234239.693418:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234264 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693424:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234239.693425:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234264 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693430:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693431:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234239.693435:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693437:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234239.693438:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693440:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234239.693442:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234239/real 1761234239] req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234239.693448:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234239.693449:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234239.693450:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234239.693451:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234239.693451:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234239.693453:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234239.693454:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234239.693458:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693462:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234239.693463:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693465:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234239.693466:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234239.693484:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693488:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234239.693491:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234239.693495:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234239.693497:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693498:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234239.693498:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693500:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234239.693501:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234239.693503:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234239.693504:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234239.693506:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234239.693508:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234239.693513:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000bb0f086 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234048:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234239.693515:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693517:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234239.693518:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000bb0f086 x1846784245234048/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234239.693521:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234239.693523:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234239.693524:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234239.693527:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234239.693528:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234239.693529:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234239.693531:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234239.693533:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000a174ebf2. 02000000:00000001:2.0:1761234239.693537:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234239.693538:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234239.693539:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000bb0f086. 00000100:00000001:2.0:1761234239.693541:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234239.693542:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693543:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693550:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234239.693551:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693552:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234239.693553:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234239.693554:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234239.693555:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693560:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234239.693560:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234239.693562:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234239.693563:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234239.693564:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234112:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234239.693566:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234239.693567:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234239.693568:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234239.693569:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234239.693571:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234239.693572:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234239.693574:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234239.693575:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234239.693577:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234112, portal 10 00000100:00000001:2.0:1761234239.693578:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234239.693579:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039814912 : -104717669736704 : ffffa0c2847e5f00) 00000100:00000040:2.0:1761234239.693582:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234264 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693586:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234239.693588:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234239.693589:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234239.693590:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245234112, offset 0 00000400:00000200:2.0:1761234239.693593:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234239.693596:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234239.693598:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234239.693599:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234239.693601:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234239.693602:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11875:11875:256:4294967295] 192.168.201.124@tcp LPNI seq info [11875:11875:8:4294967295] 00000400:00000200:2.0:1761234239.693608:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234239.693621:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234239.693623:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234239.693625:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234239.693626:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761234239.693629:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234239.693630:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1761234239.693633:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234239.693634:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234239.693638:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234239.693641:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234239.693643:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234239.693643:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234239.693645:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234239.693646:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234264 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693651:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234239.693652:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234264 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693655:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693656:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234239.693658:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693659:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234239.693660:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693661:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234239.693663:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234239/real 1761234239] req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234239.693667:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234239.693668:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234239.693669:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234239.693670:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234239.693670:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234239.693671:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234239.693672:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234239.693677:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693680:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234239.693681:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693682:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234239.693683:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234239.693684:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693688:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234239.693690:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234239.693694:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234239.693695:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693696:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234239.693696:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693697:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234239.693698:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234239.693699:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234239.693700:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234239.693701:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234239.693703:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234239.693707:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000632be758 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234112:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234239.693709:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234239.693710:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693713:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234239.693714:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234239.693715:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234239.693716:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234239.693718:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002b682c08 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234176:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234239.693720:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234239.693720:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234239.693721:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234239.693722:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234239.693723:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000dd1dcb0e. 02000000:00000001:2.0:1761234239.693725:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234239.693726:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234239.693728:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234239.693729:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234176, portal 4 00000100:00000001:2.0:1761234239.693730:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234239.693731:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039801536 : -104717669750080 : ffffa0c2847e2ac0) 00000100:00000040:2.0:1761234239.693734:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234264 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693737:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234239.693739:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234239.693740:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234239.693741:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245234176, offset 0 00000400:00000200:2.0:1761234239.693743:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234239.693746:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234239.693748:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234239.693749:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234239.693751:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234239.693752:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11876:11876:256:4294967295] 192.168.201.124@tcp LPNI seq info [11876:11876:8:4294967295] 00000400:00000200:2.0:1761234239.693757:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234239.693759:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234239.693761:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234239.693763:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234239.693764:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000200:2.1:1761234239.693766:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234239.693768:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 43 secs later) 00000800:00000100:2.0:1761234239.693770:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234239.693771:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234239.693772:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234239.693774:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234239.693776:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234239.693776:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234239.693778:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234239.693779:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234264 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693783:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234239.693784:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234264 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693788:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693789:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234239.693790:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693791:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234239.693792:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693793:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234239.693794:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234239/real 1761234239] req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234239.693798:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234239.693798:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234239.693799:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234239.693800:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234239.693800:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234239.693801:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234239.693802:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234239.693807:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693810:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234239.693811:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693812:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234239.693812:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234239.693814:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234239.693817:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234239.693819:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234239.693822:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234239.693823:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693824:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234239.693825:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693826:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234239.693827:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234239.693828:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234239.693829:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234239.693830:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234239.693831:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234239.693835:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002b682c08 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234176:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234239.693837:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693838:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234239.693839:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000632be758 x1846784245234112/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234239.693842:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234239.693873:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234239.693874:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234239.693877:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234239.693878:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234239.693879:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234239.693881:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234239.693882:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000f43b35ab. 02000000:00000001:2.0:1761234239.693884:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234239.693885:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234239.693886:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000632be758. 00000100:00000001:2.0:1761234239.693888:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234239.693889:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693891:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234239.693893:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002b682c08 x1846784245234176/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234264 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234239.693899:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234239.693901:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234239.693902:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000dd1dcb0e. 02000000:00000001:2.0:1761234239.693905:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234239.693907:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234239.693907:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234239.693909:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234239.693909:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d13d0f57. 02000000:00000001:2.0:1761234239.693911:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234239.693911:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234239.693912:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002b682c08. 00000100:00000001:2.0:1761234239.693914:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234239.693915:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693916:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234239.693919:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234239.693920:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234239.693921:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234239.693922:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234240.204998:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234240.205005:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234240.205007:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234240.205012:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234240.205017:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234240.205022:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234240.205025:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234240.205027:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234240.205029:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234240.588948:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234240.588953:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234240.588960:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234240.588962:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234240.588966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234240.588967:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234240.589060:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234240.589063:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234240.589066:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234241.228931:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234241.228935:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234241.228936:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234241.228939:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234241.228944:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234241.228947:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234241.228949:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234241.228951:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234241.228953:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.612985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234241.612990:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234241.612992:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234241.612994:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.612999:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234241.613001:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613002:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613004:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613008:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234241.613009:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234241.613010:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234241.613014:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234241.613016:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234241.613016:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613018:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234241.613019:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613021:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613022:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613025:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234241.613027:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234241.613028:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234241.613030:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234241.613030:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234241.613031:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613032:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234241.613032:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613033:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234241.613034:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234241.613129:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234241.613132:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234241.613134:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234242.252935:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234242.252945:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234242.252947:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234242.252953:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234242.252960:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234242.252964:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234242.252967:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234242.252969:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234242.252972:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234242.636950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234242.636954:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234242.636959:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234242.636960:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234242.636963:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234242.636964:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234242.637043:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234242.637049:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234242.637054:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234243.276969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234243.276975:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234243.276976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234243.276980:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234243.276986:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234243.276990:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234243.276992:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234243.276994:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234243.276996:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234243.660931:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234243.660936:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234243.660940:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234243.660941:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234243.660944:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234243.660945:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234243.660996:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234243.660999:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234243.661002:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234244.300964:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234244.300969:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234244.300971:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234244.300975:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234244.300979:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234244.300982:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234244.300984:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234244.300985:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234244.300988:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234244.684931:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234244.684935:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234244.684942:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234244.684943:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234244.684946:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234244.684947:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234244.685017:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234244.685021:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234244.685024:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234244.813031:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234244.813040:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234244.813042:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234244.813044:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234244.813047:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234244.813051:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234244.813057:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6713 00000100:00080000:0.0:1761234244.813059:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234244.813061:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813063:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234244.813064:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813066:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234244.813069:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234244.813070:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234244.813072:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234244.813075:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234244.813076:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234244.813078:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234244.813080:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813081:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234244.813082:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813083:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234244.813085:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234244.813087:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813088:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234244.813090:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234244.813093:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234244.813095:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813096:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234244.813097:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234244.813105:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761234244.813107:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234244.813111:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234244.813111:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813114:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234244.813116:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234244.813119:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234244.813125:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761234244.813129:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234244.813135:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234244.813142:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234244.813143:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234244.813146:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234244.813163:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813165:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234244.813166:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234244.813169:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234244.813169:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234244.813170:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234244.813171:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234244.813173:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234244.813174:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6713 00000100:00080000:0.0:1761234244.813175:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234244.813177:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813178:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234244.813178:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813179:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234244.813181:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234244.813182:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234244.813183:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234244.813184:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234244.813185:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234244.813187:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234244.813188:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813190:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234244.813192:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813193:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234244.813195:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234244.813196:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813198:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813199:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1761234244.813200:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1761234244.813204:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1761234244.813205:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234244.813206:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1761234244.813207:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813210:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234244.813212:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000040:2.0:1761234244.813212:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000008:00000001:0.0:1761234244.813216:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813219:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234244.813223:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000100:00000001:2.0:1761234244.813225:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1761234244.813226:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:2.0:1761234244.813227:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234244.813228:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234244.813230:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813232:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234244.813233:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234244.813233:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1761234244.813235:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234244.813236:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234244.813240:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00100000:2.0:1761234244.813240:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000080840a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234240:192.168.201.124@tcp:250: 00000100:00000001:0.0:1761234244.813244:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813246:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234244.813249:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1761234244.813250:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234244.813251:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234244.813255:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1761234244.813257:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234244.813258:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234244.813259:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 00000100:00000040:0.0:1761234244.813260:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234244.813263:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813270:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234244.813270:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00000001:0.0:1761234244.813271:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234244.813274:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234244.813276:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234244.813277:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000400:00000010:2.0:1761234244.813277:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000001:0.0:1761234244.813280:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234244.813281:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234244.813282:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234244.813284:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6713 00000100:00000200:2.0:1761234244.813284:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234240, portal 25 00000100:00080000:0.0:1761234244.813287:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1761234244.813287:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234244.813289:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246309120 : -104717463242496 : ffffa0c290cd3900) 00000100:00000040:0.0:1761234244.813291:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813294:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234244.813298:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234269 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:0.0:1761234244.813299:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813301:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234244.813304:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234244.813305:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234244.813307:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234244.813308:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000001:2.0:1761234244.813311:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000001:0.0:1761234244.813312:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761234244.813314:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234244.813317:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000040:0.0:1761234244.813320:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000010:2.0:1761234244.813321:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000001:0.0:1761234244.813322:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234244.813325:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000200:2.0:1761234244.813325:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245234240, offset 0 00000100:00000001:0.0:1761234244.813329:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234244.813330:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761234244.813333:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000200:2.0:1761234244.813336:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234244.813337:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234244.813339:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234244.813341:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234244.813342:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813343:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234244.813344:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234244.813346:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813347:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234244.813349:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000010:0.0:1761234244.813351:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761234244.813354:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234244.813356:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234244.813358:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234244.813358:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000001:0.0:1761234244.813361:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234244.813362:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1761234244.813363:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1761234244.813364:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234244.813367:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 02000000:00000010:0.0:1761234244.813368:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761234244.813387:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234244.813391:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234244.813395:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234244.813396:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234244.813399:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234244.813401:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11877:11877:256:4294967295] 192.168.201.124@tcp LPNI seq info [11877:11877:8:4294967295] 00000100:00000001:0.0:1761234244.813412:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234244.813413:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761234244.813417:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6723) 00000400:00000200:2.0:1761234244.813419:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234244.813425:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234244.813429:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234244.813435:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234244.813438:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761234244.813443:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234244.813446:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1761234244.813449:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234244.813451:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234244.813456:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234244.813459:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234244.813462:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234244.813463:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234244.813467:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234244.813469:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234269 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813476:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234244.813477:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234269 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813482:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813483:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234244.813486:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813488:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234244.813490:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813491:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234244.813493:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234244/real 1761234244] req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234244.813500:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234244.813501:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234244.813502:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234244.813503:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234244.813504:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234244.813505:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234244.813506:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234244.813511:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813515:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234244.813516:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813517:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234244.813519:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234244.813520:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813525:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234244.813527:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234244.813532:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234244.813534:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813534:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234244.813535:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813536:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234244.813537:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234244.813539:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234244.813541:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234244.813542:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234244.813544:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234244.813548:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000080840a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234240:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234244.813551:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813552:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234244.813553:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000080840a x1846784245234240/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234244.813557:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234244.813559:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234244.813560:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234244.813562:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234244.813563:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234244.813564:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234244.813566:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234244.813568:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000007c64e188. 02000000:00000001:2.0:1761234244.813570:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234244.813571:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234244.813572:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000080840a. 00000100:00000001:2.0:1761234244.813574:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234244.813575:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813576:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813581:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234244.813583:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813584:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234244.813585:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234244.813586:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234244.813587:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813592:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234244.813593:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234244.813594:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234244.813595:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234244.813597:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000efaa8dcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234304:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234244.813599:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234244.813600:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234244.813601:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234244.813602:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234244.813603:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234244.813604:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234244.813606:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234244.813608:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234244.813610:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234304, portal 10 00000100:00000001:2.0:1761234244.813611:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234244.813612:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246318848 : -104717463232768 : ffffa0c290cd5f00) 00000100:00000040:2.0:1761234244.813615:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234269 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813619:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234244.813621:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234244.813622:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234244.813623:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245234304, offset 0 00000400:00000200:2.0:1761234244.813626:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234244.813629:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234244.813632:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234244.813633:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234244.813635:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234244.813636:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11878:11878:256:4294967295] 192.168.201.124@tcp LPNI seq info [11878:11878:8:4294967295] 00000400:00000200:2.0:1761234244.813642:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234244.813645:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234244.813646:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234244.813649:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234244.813650:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761234244.813653:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234244.813654:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1761234244.813656:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234244.813657:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234244.813659:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234244.813661:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234244.813663:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234244.813664:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234244.813665:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234244.813666:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234269 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813670:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234244.813671:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234269 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813675:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813676:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234244.813677:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813678:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234244.813679:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813680:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234244.813681:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234244/real 1761234244] req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234244.813685:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234244.813686:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234244.813687:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234244.813688:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234244.813689:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234244.813690:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234244.813691:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234244.813695:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813698:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234244.813699:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813700:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234244.813701:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234244.813702:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813705:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234244.813707:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234244.813710:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234244.813711:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813712:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234244.813713:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813714:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234244.813715:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234244.813716:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234244.813717:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234244.813718:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234244.813720:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234244.813724:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000efaa8dcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234304:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234244.813728:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234244.813729:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813733:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234244.813734:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234244.813735:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234244.813736:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234244.813737:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000039bf264 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234368:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234244.813739:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234244.813740:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234244.813740:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234244.813742:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234244.813743:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234244.813744:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234244.813746:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234244.813748:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234244.813749:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234368, portal 4 00000100:00000001:2.0:1761234244.813750:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234244.813751:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246311552 : -104717463240064 : ffffa0c290cd4280) 00000100:00000040:2.0:1761234244.813753:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234269 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813757:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234244.813758:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234244.813759:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234244.813760:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245234368, offset 0 00000400:00000200:2.0:1761234244.813763:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234244.813765:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234244.813767:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234244.813768:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234244.813769:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234244.813771:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11879:11879:256:4294967295] 192.168.201.124@tcp LPNI seq info [11879:11879:8:4294967295] 00000400:00000200:2.0:1761234244.813776:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234244.813779:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234244.813780:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234244.813783:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234244.813784:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000200:2.1:1761234244.813786:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234244.813787:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 38 secs later) 00000800:00000100:2.0:1761234244.813789:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234244.813790:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234244.813791:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234244.813793:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234244.813794:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234244.813795:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234244.813796:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234244.813798:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234269 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813807:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234244.813809:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234269 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813812:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813813:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234244.813815:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813816:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234244.813817:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813818:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234244.813820:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234244/real 1761234244] req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234244.813824:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234244.813825:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234244.813825:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234244.813826:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234244.813827:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234244.813828:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234244.813829:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234244.813837:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813840:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234244.813841:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813842:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234244.813864:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234244.813867:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234244.813870:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234244.813872:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234244.813876:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234244.813877:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813878:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234244.813879:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813880:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234244.813880:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234244.813882:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234244.813883:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234244.813884:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234244.813885:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234244.813890:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000039bf264 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234368:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234244.813892:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813893:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234244.813894:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000efaa8dcd x1846784245234304/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234244.813898:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234244.813899:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234244.813900:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234244.813902:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234244.813903:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234244.813903:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234244.813905:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234244.813905:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000005851ba37. 02000000:00000001:2.0:1761234244.813907:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234244.813908:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234244.813908:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000efaa8dcd. 00000100:00000001:2.0:1761234244.813910:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234244.813911:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813911:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234244.813912:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000039bf264 x1846784245234368/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234269 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234244.813916:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234244.813917:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234244.813918:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234244.813919:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234244.813920:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234244.813920:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234244.813921:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234244.813922:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:2.0:1761234244.813923:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234244.813923:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234244.813924:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000039bf264. 00000100:00000001:2.0:1761234244.813925:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234244.813926:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813926:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234244.813928:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234244.813929:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234244.813930:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234244.813931:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234245.324914:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234245.324917:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234245.324919:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234245.324922:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234245.324926:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234245.324928:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234245.324930:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234245.324932:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234245.324933:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234245.708935:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234245.708939:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234245.708945:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234245.708946:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234245.708949:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234245.708950:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234245.709022:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234245.709025:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234245.709029:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1761234246.348979:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1761234246.348986:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1761234246.348987:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234246.348992:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:3.0:1761234246.348998:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234246.349002:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234246.349004:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234246.349007:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1761234246.349009:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234246.733006:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234246.733013:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234246.733024:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234246.733026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234246.733030:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234246.733031:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234246.733083:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234246.733086:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234246.733089:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234247.372996:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234247.373000:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234247.373001:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234247.373005:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234247.373008:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234247.373011:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234247.373013:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234247.373014:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234247.373016:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234247.756956:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234247.756960:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234247.756966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234247.756967:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234247.756969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234247.756970:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234247.757054:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234247.757058:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234247.757061:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234248.396932:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234248.396936:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234248.396937:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234248.396941:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234248.396945:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234248.396947:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234248.396949:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234248.396951:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234248.396953:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234248.780938:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234248.780942:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234248.780947:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234248.780948:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234248.780950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234248.780951:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234248.780998:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234248.781002:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234248.781004:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234249.420970:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234249.420977:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234249.420978:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234249.420984:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234249.420990:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234249.420995:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.420999:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234249.421002:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.421005:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234249.804951:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234249.804956:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234249.804961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234249.804962:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234249.804965:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234249.804966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234249.805064:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234249.805067:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234249.805071:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234249.932978:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234249.932998:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234249.932999:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234249.933002:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234249.933004:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234249.933011:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234249.933017:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6718 00000100:00080000:3.0:1761234249.933019:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234249.933021:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933023:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234249.933025:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933027:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234249.933030:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234249.933031:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234249.933033:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234249.933036:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234249.933037:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234249.933039:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234249.933040:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933041:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234249.933042:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933043:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234249.933045:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234249.933048:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933049:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234249.933051:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234249.933053:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234249.933055:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933057:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234249.933058:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234249.933066:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000108ab0a5. 00000020:00000040:3.0:1761234249.933068:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234249.933071:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234249.933072:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933074:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234249.933076:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234249.933079:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234249.933087:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074003a94. 00000100:00000001:3.0:1761234249.933092:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234249.933097:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234249.933103:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234249.933105:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234249.933119:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000108ab0a5] to pc [ptlrpcd_rcv+-1] req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234249.933134:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933136:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234249.933137:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234249.933140:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234249.933140:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234249.933141:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234249.933142:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234249.933143:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234249.933144:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6718 00000100:00080000:3.0:1761234249.933146:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234249.933147:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933149:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234249.933150:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933151:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234249.933152:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234249.933153:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234249.933154:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234249.933155:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234249.933156:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234249.933157:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234249.933159:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933160:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234249.933161:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933162:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234249.933163:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234249.933166:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933167:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234249.933169:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234249.933173:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234249.933175:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933176:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234249.933178:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1761234249.933179:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234249.933182:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933183:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234249.933184:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933185:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000010:3.0:1761234249.933188:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f61de4c8. 00000020:00000040:3.0:1761234249.933190:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000040:2.0:1761234249.933192:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234249.933193:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234249.933194:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933197:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234249.933198:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234249.933199:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234249.933203:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074a27ae0. 00000100:00000001:2.0:1761234249.933204:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234249.933206:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933207:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234249.933210:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1761234249.933212:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234249.933225:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234249.933230:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234432:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234249.933230:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234249.933231:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933236:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1761234249.933237:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f61de4c8] to pc [ptlrpcd_rcv+-1] req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234249.933239:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234249.933241:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933245:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234249.933246:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1761234249.933248:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234249.933249:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:2.0:1761234249.933251:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 00000100:00000001:3.0:1761234249.933252:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234249.933253:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1761234249.933254:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933255:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234249.933257:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234249.933259:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000400:00000010:2.0:1761234249.933260:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00080000:3.0:1761234249.933261:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6718 00000100:00080000:3.0:1761234249.933263:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234249.933264:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933265:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234249.933267:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933269:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234249.933270:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234249.933271:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761234249.933271:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000400:00000010:2.0:1761234249.933272:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000020:00000040:3.0:1761234249.933273:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:3.0:1761234249.933298:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000200:2.0:1761234249.933300:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234432, portal 25 00000020:00000001:3.0:1761234249.933300:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:3.0:1761234249.933303:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234249.933304:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234249.933306:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1761234249.933307:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091110144 : -104717618441472 : ffffa0c2878d1300) 00000100:00000040:3.0:1761234249.933309:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933312:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234249.933314:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00000040:2.0:1761234249.933315:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234274 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00080000:3.0:1761234249.933318:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234249.933322:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234249.933324:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234249.933326:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234249.933327:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234249.933330:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 02000000:00000001:3.0:1761234249.933330:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933332:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000400:00000010:2.0:1761234249.933334:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000008:00100000:3.0:1761234249.933334:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000200:2.0:1761234249.933337:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245234432, offset 0 00000008:00000001:3.0:1761234249.933338:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933342:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234249.933346:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000010:3.0:1761234249.933346:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002431276f. 00000020:00000040:3.0:1761234249.933348:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:3.0:1761234249.933351:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234249.933352:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933353:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234249.933354:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234249.933355:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234249.933357:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000001c1b0e94. 00000100:00000001:3.0:1761234249.933359:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234249.933361:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00100000:3.0:1761234249.933364:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234249.933369:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000001:3.0:1761234249.933373:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234249.933374:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234249.933376:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1761234249.933379:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002431276f] to pc [ptlrpcd_rcv+-1] req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234249.933381:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234249.933386:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11880:11880:256:4294967295] 192.168.201.124@tcp LPNI seq info [11880:11880:8:4294967295] 00000100:00000001:3.0:1761234249.933394:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234249.933396:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:3.0:1761234249.933399:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6729) 00000400:00000200:2.0:1761234249.933401:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234249.933407:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234249.933411:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234249.933417:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234249.933419:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761234249.933424:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234249.933426:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1761234249.933429:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234249.933431:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234249.933435:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234249.933438:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234249.933442:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234249.933443:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234249.933447:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234249.933449:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234274 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933456:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234249.933457:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234274 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933461:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933463:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234249.933467:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933468:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234249.933470:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933472:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234249.933474:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234249/real 1761234249] req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234249.933479:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234249.933481:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234249.933481:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234249.933483:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234249.933484:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234249.933485:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234249.933486:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234249.933491:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933494:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234249.933495:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933496:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234249.933498:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234249.933499:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933503:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234249.933506:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234249.933509:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234249.933511:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933512:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234249.933512:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933514:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234249.933515:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234249.933517:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234249.933519:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234249.933521:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234249.933523:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234249.933528:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000108ab0a5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234432:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234249.933530:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933531:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234249.933533:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000108ab0a5 x1846784245234432/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234249.933536:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234249.933538:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234249.933539:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234249.933541:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234249.933542:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234249.933543:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234249.933544:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234249.933545:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074003a94. 02000000:00000001:2.0:1761234249.933547:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234249.933548:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234249.933549:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000108ab0a5. 00000100:00000001:2.0:1761234249.933552:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234249.933552:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933553:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933559:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234249.933560:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933561:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234249.933562:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234249.933563:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234249.933564:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933569:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234249.933569:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234249.933571:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234249.933572:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234249.933573:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234496:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234249.933575:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234249.933576:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234249.933577:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234249.933578:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234249.933580:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234249.933581:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234249.933583:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234249.933585:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234249.933586:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234496, portal 10 00000100:00000001:2.0:1761234249.933587:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234249.933588:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091124736 : -104717618426880 : ffffa0c2878d4c00) 00000100:00000040:2.0:1761234249.933592:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234274 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933595:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234249.933597:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234249.933598:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234249.933599:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245234496, offset 0 00000400:00000200:2.0:1761234249.933601:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234249.933604:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234249.933606:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234249.933607:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234249.933609:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234249.933610:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11881:11881:256:4294967295] 192.168.201.124@tcp LPNI seq info [11881:11881:8:4294967295] 00000400:00000200:2.0:1761234249.933615:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234249.933618:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234249.933619:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234249.933622:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234249.933623:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761234249.933626:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234249.933627:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1761234249.933629:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234249.933631:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234249.933632:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234249.933634:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234249.933636:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234249.933637:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234249.933638:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234249.933639:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234274 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933643:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234249.933644:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234274 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933648:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933649:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234249.933650:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933651:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234249.933652:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933653:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234249.933655:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234249/real 1761234249] req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234249.933659:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234249.933660:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234249.933660:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234249.933661:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234249.933662:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234249.933663:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234249.933664:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234249.933668:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933671:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234249.933671:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933672:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234249.933673:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234249.933675:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933678:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234249.933680:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234249.933684:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234249.933685:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933686:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234249.933686:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933687:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234249.933688:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234249.933689:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234249.933690:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234249.933691:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234249.933693:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234249.933696:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f61de4c8 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234496:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234249.933698:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234249.933699:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933703:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234249.933704:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234249.933705:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234249.933706:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234249.933707:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234560:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234249.933709:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234249.933710:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234249.933710:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234249.933711:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234249.933713:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234249.933714:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234249.933716:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234249.933717:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234249.933718:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234560, portal 4 00000100:00000001:2.0:1761234249.933719:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234249.933720:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091128384 : -104717618423232 : ffffa0c2878d5a40) 00000100:00000040:2.0:1761234249.933723:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234274 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933726:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234249.933728:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234249.933729:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234249.933730:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245234560, offset 0 00000400:00000200:2.0:1761234249.933733:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234249.933735:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234249.933736:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234249.933738:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234249.933739:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234249.933740:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11882:11882:256:4294967295] 192.168.201.124@tcp LPNI seq info [11882:11882:8:4294967295] 00000400:00000200:2.0:1761234249.933745:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234249.933748:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234249.933749:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234249.933751:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234249.933752:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000200:2.1:1761234249.933754:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234249.933756:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 32 secs later) 00000800:00000100:2.0:1761234249.933758:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234249.933759:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234249.933761:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234249.933762:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234249.933764:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234249.933765:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234249.933766:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234249.933767:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234274 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933771:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234249.933773:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234274 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933776:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933777:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234249.933778:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933779:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234249.933780:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933781:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234249.933782:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234249/real 1761234249] req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234249.933786:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234249.933787:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234249.933787:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234249.933788:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234249.933789:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234249.933790:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234249.933791:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234249.933795:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933798:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234249.933799:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933800:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234249.933800:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234249.933802:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234249.933805:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234249.933806:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234249.933814:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234249.933815:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933816:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234249.933816:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933817:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234249.933818:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234249.933819:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234249.933820:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234249.933821:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234249.933823:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234249.933828:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002431276f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234560:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234249.933831:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933832:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234249.933834:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f61de4c8 x1846784245234496/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234249.933840:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234249.933841:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234249.933842:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234249.933870:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234249.933871:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234249.933873:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234249.933876:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234249.933877:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074a27ae0. 02000000:00000001:2.0:1761234249.933881:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234249.933882:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234249.933884:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f61de4c8. 00000100:00000001:2.0:1761234249.933888:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234249.933889:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933891:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234249.933895:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002431276f x1846784245234560/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234274 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234249.933902:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234249.933903:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234249.933904:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234249.933907:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234249.933908:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234249.933909:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234249.933910:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234249.933911:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000001c1b0e94. 02000000:00000001:2.0:1761234249.933924:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234249.933925:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234249.933926:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002431276f. 00000100:00000001:2.0:1761234249.933927:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234249.933928:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933929:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234249.933931:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234249.933932:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234249.933933:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234249.933934:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234250.444953:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234250.444958:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234250.444959:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234250.444963:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234250.444966:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234250.444969:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234250.444971:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234250.444973:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234250.444975:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234250.828940:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234250.828946:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234250.828953:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234250.828955:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234250.828960:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234250.828961:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234250.829039:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234250.829042:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234250.829044:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234251.468979:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234251.468985:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234251.468999:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234251.469005:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234251.469011:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234251.469014:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234251.469017:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234251.469019:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234251.469021:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852954:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234251.852959:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234251.852961:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234251.852964:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852969:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234251.852971:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852972:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852974:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852976:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234251.852978:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234251.852978:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234251.852982:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234251.852983:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234251.852984:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852985:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234251.852986:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852987:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852988:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234251.852991:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234251.852991:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234251.852993:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234251.852993:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234251.852994:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852995:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234251.852995:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852996:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234251.852997:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234251.853063:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234251.853066:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234251.853069:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234252.492971:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234252.492975:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234252.492976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234252.492980:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234252.492984:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234252.492988:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234252.492989:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234252.492991:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234252.492993:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234252.876942:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234252.876946:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234252.876951:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234252.876952:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234252.876954:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234252.876955:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234252.877078:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234252.877082:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234252.877086:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234253.516969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234253.516975:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234253.516977:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234253.516980:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234253.516985:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234253.516989:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234253.516991:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234253.516993:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234253.516995:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234253.900970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234253.900975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234253.900982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234253.900983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234253.900987:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234253.900988:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234253.901057:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234253.901060:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234253.901062:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234254.540944:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234254.540948:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234254.540949:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234254.540953:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234254.540956:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234254.540959:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234254.540961:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234254.540963:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234254.540965:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234254.924966:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234254.924969:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234254.924974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234254.924975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234254.924978:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234254.924979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234254.925044:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234254.925047:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234254.925050:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1761234255.053042:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234255.053051:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234255.053052:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:1.0:1761234255.053054:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234255.053056:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234255.053060:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234255.053066:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6724 00000100:00080000:1.0:1761234255.053068:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234255.053070:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053072:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234255.053073:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053075:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234255.053078:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234255.053079:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:1.0:1761234255.053081:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234255.053084:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:1.0:1761234255.053085:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:1.0:1761234255.053087:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:1.0:1761234255.053088:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053089:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234255.053090:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053092:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234255.053094:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:1.0:1761234255.053097:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053098:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234255.053101:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234255.053103:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234255.053105:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053106:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:1.0:1761234255.053107:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234255.053116:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a014c997. 00000020:00000040:1.0:1761234255.053118:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:1.0:1761234255.053121:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234255.053122:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053124:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234255.053127:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234255.053129:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234255.053136:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000006d8d01a9. 00000100:00000001:1.0:1761234255.053140:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234255.053146:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234255.053153:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1761234255.053154:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234255.053157:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a014c997] to pc [ptlrpcd_rcv+-1] req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234255.053173:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053174:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234255.053175:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1761234255.053190:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234255.053191:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1761234255.053192:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234255.053193:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234255.053194:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:1.0:1761234255.053196:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6724 00000100:00080000:1.0:1761234255.053197:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1761234255.053199:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053200:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234255.053201:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053202:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:1.0:1761234255.053204:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234255.053204:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:1.0:1761234255.053205:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234255.053206:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:1.0:1761234255.053207:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:1.0:1761234255.053208:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:1.0:1761234255.053210:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053210:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234255.053211:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053212:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:1.0:1761234255.053214:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:1.0:1761234255.053215:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053217:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234255.053218:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1761234255.053222:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234255.053223:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053225:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234255.053227:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234255.053230:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053231:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1761234255.053233:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004567def3. 00000020:00000040:1.0:1761234255.053235:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000001:2.0:1761234255.053237:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1761234255.053238:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234255.053239:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053241:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234255.053242:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1761234255.053243:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234255.053246:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.053246:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 02000000:00000010:1.0:1761234255.053248:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000008bad38d7. 00000100:00000001:1.0:1761234255.053252:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234255.053255:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00100000:1.0:1761234255.053257:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234255.053285:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234255.053285:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1761234255.053287:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.053289:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234255.053291:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004567def3] to pc [ptlrpcd_rcv+-1] req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234255.053295:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1761234255.053297:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234255.053299:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053300:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:1.0:1761234255.053302:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1761234255.053306:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234624:192.168.201.124@tcp:250: 00000100:00000001:1.0:1761234255.053307:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:1.0:1761234255.053308:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1761234255.053310:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1761234255.053311:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1761234255.053314:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:2.0:1761234255.053314:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1761234255.053318:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6724 00000100:00080000:1.0:1761234255.053320:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1761234255.053320:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1761234255.053323:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000001:2.0:1761234255.053324:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053325:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234255.053327:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053329:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 02000000:00000001:2.0:1761234255.053329:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1761234255.053332:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:1.0:1761234255.053332:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:1.0:1761234255.053333:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:1.0:1761234255.053334:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:1.0:1761234255.053336:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000010:2.0:1761234255.053336:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 00000020:00000001:1.0:1761234255.053337:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:2.0:1761234255.053340:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234255.053342:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053345:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1761234255.053346:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053348:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000400:00000010:2.0:1761234255.053348:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000020:00000040:1.0:1761234255.053353:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:1.0:1761234255.053355:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053356:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1761234255.053357:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1761234255.053357:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 02000000:00000001:1.0:1761234255.053360:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1761234255.053362:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053364:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:1.0:1761234255.053365:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1761234255.053366:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053367:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234255.053367:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234624, portal 25 00000100:00000010:1.0:1761234255.053373:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000f33bf85c. 00000100:00000001:2.0:1761234255.053373:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000020:00000040:1.0:1761234255.053376:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000001:2.0:1761234255.053376:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039807616 : -104717669744000 : ffffa0c2847e4280) 02000000:00000001:1.0:1761234255.053379:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:1.0:1761234255.053380:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1761234255.053382:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1761234255.053383:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1761234255.053384:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1761234255.053387:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000c6b52d7d. 00000100:00000040:2.0:1761234255.053387:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234280 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234255.053390:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1761234255.053396:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:1.0:1761234255.053410:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234255.053410:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:1.0:1761234255.053413:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1761234255.053416:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000f33bf85c] to pc [ptlrpcd_rcv+-1] req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234255.053420:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:1.0:1761234255.053425:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234255.053425:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000001:1.0:1761234255.053426:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:1.0:1761234255.053430:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6734) 00000100:00000200:2.0:1761234255.053431:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245234624, offset 0 00000400:00000200:2.0:1761234255.053442:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234255.053457:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234255.053465:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234255.053469:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234255.053472:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234255.053477:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11883:11883:256:4294967295] 192.168.201.124@tcp LPNI seq info [11883:11883:8:4294967295] 00000400:00000200:2.0:1761234255.053489:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234255.053496:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234255.053501:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234255.053508:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234255.053512:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761234255.053518:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234255.053521:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1761234255.053525:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234255.053527:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234255.053534:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234255.053538:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234255.053543:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234255.053545:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234255.053550:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234255.053554:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234280 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053563:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234255.053566:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234280 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053574:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.053576:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234255.053581:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.053583:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234255.053586:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.053589:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234255.053592:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234255/real 1761234255] req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234255.053602:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234255.053604:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234255.053605:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234255.053608:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234255.053609:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234255.053611:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234255.053614:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234255.053622:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053629:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234255.053631:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.053633:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234255.053635:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234255.053639:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053646:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234255.053650:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234255.053657:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234255.053660:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.053661:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234255.053662:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.053665:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234255.053666:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234255.053669:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234255.053672:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234255.053674:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234255.053678:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234255.053685:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a014c997 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234624:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234255.053690:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.053692:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234255.053694:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a014c997 x1846784245234624/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234255.053706:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234255.053709:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234255.053711:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234255.053714:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234255.053715:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234255.053717:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234255.053719:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234255.053721:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000006d8d01a9. 02000000:00000001:2.0:1761234255.053725:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234255.053726:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234255.053727:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a014c997. 00000100:00000001:2.0:1761234255.053730:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234255.053731:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.053733:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.053740:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234255.053742:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.053744:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234255.053745:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234255.053747:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234255.053750:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053756:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234255.053757:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234255.053759:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234255.053760:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234255.053763:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234688:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234255.053766:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234255.053768:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234255.053769:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234255.053771:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234255.053773:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234255.053775:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234255.053778:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234255.053781:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234255.053783:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234688, portal 10 00000100:00000001:2.0:1761234255.053786:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234255.053786:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039812480 : -104717669739136 : ffffa0c2847e5580) 00000100:00000040:2.0:1761234255.053792:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234280 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053798:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234255.053801:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234255.053803:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234255.053805:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245234688, offset 0 00000400:00000200:2.0:1761234255.053809:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234255.053815:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234255.053819:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234255.053821:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234255.053823:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234255.053826:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11884:11884:256:4294967295] 192.168.201.124@tcp LPNI seq info [11884:11884:8:4294967295] 00000400:00000200:2.0:1761234255.053834:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234255.053839:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234255.053843:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234255.053880:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234255.053883:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761234255.053887:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234255.053889:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1761234255.053893:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234255.053894:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234255.053898:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234255.053901:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234255.053904:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234255.053905:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234255.053908:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234255.053911:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234280 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053919:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234255.053921:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234280 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053932:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.053934:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234255.053937:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.053939:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234255.053941:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.053943:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234255.053946:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234255/real 1761234255] req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234255.053953:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234255.053955:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234255.053956:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234255.053958:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234255.053959:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234255.053961:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234255.053963:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234255.053970:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053977:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234255.053978:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.053980:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234255.053981:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234255.053984:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.053990:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234255.053993:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234255.054000:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234255.054002:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.054004:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234255.054005:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.054007:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234255.054009:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234255.054011:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234255.054014:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234255.054016:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234255.054019:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234255.054027:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004567def3 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234688:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234255.054030:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234255.054033:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.054038:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234255.054039:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234255.054041:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234255.054043:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234255.054045:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234752:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234255.054048:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234255.054050:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234255.054052:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234255.054053:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234255.054056:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000003562879b. 02000000:00000001:2.0:1761234255.054058:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234255.054061:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234255.054063:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234255.054066:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234752, portal 4 00000100:00000001:2.0:1761234255.054068:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234255.054069:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356039808832 : -104717669742784 : ffffa0c2847e4740) 00000100:00000040:2.0:1761234255.054074:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234280 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.054081:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234255.054083:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234255.054086:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234255.054088:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245234752, offset 0 00000400:00000200:2.0:1761234255.054092:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234255.054097:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234255.054101:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234255.054103:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234255.054105:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234255.054108:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11885:11885:256:4294967295] 192.168.201.124@tcp LPNI seq info [11885:11885:8:4294967295] 00000400:00000200:2.0:1761234255.054115:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234255.054120:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234255.054123:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234255.054127:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234255.054129:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000200:2.1:1761234255.054133:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234255.054135:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 27 secs later) 00000800:00000100:2.0:1761234255.054139:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234255.054141:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234255.054144:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234255.054147:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234255.054150:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234255.054151:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234255.054154:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234255.054156:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234280 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.054164:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234255.054167:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234280 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.054174:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.054175:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234255.054199:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.054201:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234255.054203:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.054205:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234255.054208:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234255/real 1761234255] req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234255.054216:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234255.054217:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234255.054219:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234255.054221:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234255.054222:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234255.054225:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234255.054227:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234255.054234:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.054241:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234255.054242:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.054243:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234255.054245:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234255.054248:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234255.054254:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234255.054257:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234255.054263:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234255.054265:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.054267:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234255.054268:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.054269:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234255.054271:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234255.054273:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234255.054275:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234255.054277:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234255.054281:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234255.054288:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000f33bf85c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234752:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234255.054291:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.054293:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234255.054295:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004567def3 x1846784245234688/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234255.054302:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234255.054303:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234255.054305:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234255.054307:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234255.054309:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234255.054310:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234255.054312:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234255.054313:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000008bad38d7. 02000000:00000001:2.0:1761234255.054315:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234255.054316:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234255.054317:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004567def3. 00000100:00000001:2.0:1761234255.054320:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234255.054321:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.054323:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234255.054325:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000f33bf85c x1846784245234752/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234280 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234255.054331:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234255.054332:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234255.054333:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000003562879b. 02000000:00000001:2.0:1761234255.054336:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234255.054337:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234255.054337:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234255.054339:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234255.054340:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000c6b52d7d. 02000000:00000001:2.0:1761234255.054342:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234255.054343:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234255.054345:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000f33bf85c. 00000100:00000001:2.0:1761234255.054347:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234255.054347:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.054349:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.054353:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234255.054354:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.054356:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234255.054357:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234255.564991:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234255.565000:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234255.565002:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234255.565007:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234255.565012:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234255.565015:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.565018:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234255.565020:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234255.565022:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234255.948989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234255.948995:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234255.949002:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234255.949004:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234255.949008:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234255.949010:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234255.949083:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234255.949086:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234255.949089:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234256.588954:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234256.588958:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234256.588960:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234256.588963:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234256.588969:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234256.588972:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234256.588975:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234256.588977:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234256.588979:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234256.972928:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234256.972932:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234256.972938:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234256.972939:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234256.972941:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234256.972942:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234256.973016:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234256.973019:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234256.973023:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234257.612992:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234257.612999:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234257.613000:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234257.613005:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234257.613010:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234257.613014:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234257.613016:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234257.613018:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234257.613020:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234257.996938:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234257.996943:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234257.996950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234257.996951:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234257.996954:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234257.996955:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234257.997056:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234257.997059:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234257.997062:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234258.636927:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234258.636932:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234258.636933:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234258.636937:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234258.636941:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234258.636944:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234258.636946:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234258.636947:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234258.636950:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234259.020970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234259.020975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234259.020982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234259.020983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234259.020986:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234259.020987:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234259.021087:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234259.021090:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234259.021093:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234259.660982:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234259.660987:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234259.660988:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234259.660993:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234259.660997:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234259.661000:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234259.661003:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234259.661005:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234259.661008:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234260.044974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234260.044978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234260.044985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234260.044986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234260.044991:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234260.044993:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234260.045055:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234260.045058:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234260.045061:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234260.173065:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234260.173088:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234260.173091:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234260.173095:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234260.173098:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234260.173105:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234260.173115:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6729 00000100:00080000:3.0:1761234260.173119:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234260.173122:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173125:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234260.173128:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173131:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234260.173136:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234260.173139:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234260.173142:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234260.173146:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234260.173148:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234260.173151:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234260.173154:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173156:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234260.173158:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173168:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234260.173170:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234260.173173:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173175:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234260.173178:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234260.173182:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234260.173184:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173186:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234260.173188:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234260.173200:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000db341c88. 00000020:00000040:3.0:1761234260.173204:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234260.173209:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234260.173210:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173213:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234260.173216:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234260.173221:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234260.173230:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000062959873. 00000100:00000001:3.0:1761234260.173237:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234260.173244:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234260.173253:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234260.173255:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234260.173258:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000db341c88] to pc [ptlrpcd_rcv+-1] req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234260.173283:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173285:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234260.173287:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234260.173290:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234260.173291:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234260.173292:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234260.173293:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234260.173295:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234260.173296:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6729 00000100:00080000:3.0:1761234260.173298:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234260.173299:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173301:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234260.173302:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173303:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234260.173305:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234260.173306:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234260.173306:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234260.173308:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234260.173309:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234260.173310:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234260.173312:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173312:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234260.173313:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173314:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234260.173315:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234260.173317:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173318:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234260.173320:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234260.173326:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234260.173328:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173329:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234260.173333:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234260.173337:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173339:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234260.173341:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000004b242e46. 00000020:00000040:3.0:1761234260.173343:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761234260.173345:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234260.173345:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173346:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234260.173347:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234260.173348:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234260.173351:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000be06bfbf. 00000100:00000001:3.0:1761234260.173353:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.173355:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1761234260.173356:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.173364:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234260.173367:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234260.173368:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.173370:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234260.173372:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000004b242e46] to pc [ptlrpcd_rcv+-1] req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234260.173378:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173380:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234260.173381:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1761234260.173387:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234260.173388:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234260.173389:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234260.173392:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234260.173393:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234260.173396:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234260.173400:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6729 00000100:00080000:3.0:1761234260.173403:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234260.173408:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173411:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234260.173414:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234260.173418:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234260.173422:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234260.173423:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761234260.173424:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234260.173426:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000001:2.0:1761234260.173429:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1761234260.173430:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:3.0:1761234260.173432:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:2.0:1761234260.173433:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234260.173442:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000001:2.0:1761234260.173443:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1761234260.173444:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234260.173449:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 02000000:00000001:2.0:1761234260.173450:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173452:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234260.173456:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:3.0:1761234260.173461:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00100000:2.0:1761234260.173462:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000db341c88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234816:192.168.201.124@tcp:250: 00000100:00000001:3.0:1761234260.173468:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234260.173472:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234260.173474:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234260.173476:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173477:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234260.173478:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234260.173479:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173482:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.173483:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000010:3.0:1761234260.173485:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000266eaa93. 00000020:00000040:3.0:1761234260.173488:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:2.0:1761234260.173490:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1761234260.173491:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234260.173494:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234260.173496:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173496:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234260.173500:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234260.173502:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234260.173503:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1761234260.173508:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000296782d7. 00000100:00000001:3.0:1761234260.173510:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234260.173514:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:2.0:1761234260.173515:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234260.173521:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173529:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234260.173530:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234260.173532:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00000040:3.0:1761234260.173534:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000266eaa93] to pc [ptlrpcd_rcv+-1] req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234260.173540:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234260.173541:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000010:2.0:1761234260.173544:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000040:3.0:1761234260.173546:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6739) 00000100:00000200:2.0:1761234260.173555:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234816, portal 25 00000100:00000001:2.0:1761234260.173560:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234260.173561:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091132032 : -104717618419584 : ffffa0c2878d6880) 00000100:00000040:2.0:1761234260.173571:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234285 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.173580:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234260.173587:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234260.173590:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234260.173593:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245234816, offset 0 00000400:00000200:2.0:1761234260.173601:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234260.173617:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234260.173624:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234260.173627:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234260.173630:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234260.173633:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11886:11886:256:4294967295] 192.168.201.124@tcp LPNI seq info [11886:11886:8:4294967295] 00000400:00000200:2.0:1761234260.173646:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234260.173651:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234260.173656:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234260.173663:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234260.173667:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761234260.173673:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234260.173676:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1761234260.173680:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234260.173683:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234260.173689:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234260.173694:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234260.173699:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234260.173701:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234260.173705:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234260.173708:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234285 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.173719:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234260.173721:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234285 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.173728:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.173729:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234260.173735:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.173737:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234260.173740:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.173742:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234260.173745:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234260/real 1761234260] req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234260.173755:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234260.173757:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234260.173759:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234260.173760:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234260.173762:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234260.173763:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234260.173766:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234260.173774:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.173780:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234260.173782:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.173784:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234260.173786:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234260.173788:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.173795:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234260.173798:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234260.173805:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234260.173808:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.173809:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234260.173810:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.173813:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234260.173815:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234260.173818:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234260.173821:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234260.173823:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234260.173826:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234260.173833:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000db341c88 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245234816:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234260.173838:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.173840:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234260.173842:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000db341c88 x1846784245234816/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234260.173881:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234260.173883:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234260.173885:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234260.173889:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234260.173890:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234260.173892:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234260.173895:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234260.173897:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062959873. 02000000:00000001:2.0:1761234260.173901:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234260.173903:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234260.173904:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000db341c88. 00000100:00000001:2.0:1761234260.173907:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234260.173908:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.173910:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.173917:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234260.173919:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.173921:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234260.173923:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234260.173924:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234260.173927:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.173934:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234260.173935:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234260.173938:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234260.173939:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234260.173942:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000004b242e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234880:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234260.173945:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234260.173947:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234260.173948:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234260.173950:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234260.173952:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234260.173954:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234260.173957:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234260.173960:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234260.173963:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234880, portal 10 00000100:00000001:2.0:1761234260.173964:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234260.173966:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091125952 : -104717618425664 : ffffa0c2878d50c0) 00000100:00000040:2.0:1761234260.173970:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234285 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.173977:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234260.173980:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234260.173982:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234260.173983:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245234880, offset 0 00000400:00000200:2.0:1761234260.173988:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234260.173993:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234260.173996:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234260.173999:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234260.174001:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234260.174003:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11887:11887:256:4294967295] 192.168.201.124@tcp LPNI seq info [11887:11887:8:4294967295] 00000400:00000200:2.0:1761234260.174012:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234260.174017:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234260.174019:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234260.174023:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234260.174025:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761234260.174029:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234260.174031:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1761234260.174035:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234260.174037:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234260.174039:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234260.174043:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234260.174045:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234260.174047:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234260.174049:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234260.174051:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234285 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174058:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234260.174060:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234285 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174066:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.174068:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234260.174070:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.174072:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234260.174073:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.174075:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234260.174077:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234260/real 1761234260] req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234260.174103:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234260.174104:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234260.174105:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234260.174107:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234260.174108:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234260.174110:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234260.174112:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234260.174120:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174125:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234260.174126:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174128:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234260.174129:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234260.174132:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174139:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234260.174141:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234260.174148:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234260.174151:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174152:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234260.174153:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174155:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234260.174157:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234260.174159:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234260.174162:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234260.174164:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234260.174166:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234260.174173:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000004b242e46 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234880:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234260.174177:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234260.174180:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174187:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234260.174188:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234260.174190:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234260.174191:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234260.174194:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000266eaa93 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234944:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234260.174197:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234260.174198:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234260.174200:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234260.174201:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234260.174204:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000070e0ffb1. 02000000:00000001:2.0:1761234260.174206:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234260.174208:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234260.174210:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234260.174213:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245234944, portal 4 00000100:00000001:2.0:1761234260.174215:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234260.174216:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091129600 : -104717618422016 : ffffa0c2878d5f00) 00000100:00000040:2.0:1761234260.174220:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234285 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174226:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234260.174229:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234260.174231:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234260.174233:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245234944, offset 0 00000400:00000200:2.0:1761234260.174237:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234260.174242:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234260.174245:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234260.174247:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234260.174249:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234260.174252:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11888:11888:256:4294967295] 192.168.201.124@tcp LPNI seq info [11888:11888:8:4294967295] 00000400:00000200:2.0:1761234260.174260:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234260.174264:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234260.174267:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234260.174271:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234260.174273:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1761234260.174277:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234260.174279:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1761234260.174282:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234260.174284:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234260.174287:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234260.174290:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234260.174293:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234260.174294:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234260.174296:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234260.174298:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234285 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174305:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234260.174307:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234285 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174313:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.174315:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234260.174317:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.174318:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234260.174320:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.174322:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234260.174324:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234260/real 1761234260] req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234260.174331:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234260.174332:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234260.174334:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234260.174335:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234260.174337:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234260.174338:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234260.174340:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234260.174347:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174353:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234260.174354:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174355:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234260.174356:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234260.174359:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234260.174365:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234260.174367:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234260.174373:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234260.174376:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174377:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234260.174378:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174380:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234260.174381:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234260.174383:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234260.174385:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234260.174387:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234260.174390:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234260.174396:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000266eaa93 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245234944:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234260.174400:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174402:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234260.174404:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000004b242e46 x1846784245234880/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234260.174410:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234260.174412:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234260.174413:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234260.174415:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234260.174416:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234260.174417:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234260.174419:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234260.174420:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000be06bfbf. 02000000:00000001:2.0:1761234260.174423:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234260.174424:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234260.174425:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000004b242e46. 00000100:00000001:2.0:1761234260.174428:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234260.174428:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174430:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234260.174432:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000266eaa93 x1846784245234944/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234285 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234260.174438:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234260.174439:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234260.174440:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000070e0ffb1. 02000000:00000001:2.0:1761234260.174444:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234260.174445:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234260.174446:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234260.174447:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234260.174448:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000296782d7. 02000000:00000001:2.0:1761234260.174450:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234260.174451:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234260.174452:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000266eaa93. 00000100:00000001:2.0:1761234260.174454:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234260.174455:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174457:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.174460:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234260.174461:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.174463:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234260.174464:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234260.684913:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234260.684917:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234260.684919:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234260.684922:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234260.684926:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234260.684929:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.684930:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234260.684932:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234260.684934:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.068990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234261.068997:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234261.069000:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234261.069003:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069010:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234261.069012:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069014:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069018:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069022:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234261.069024:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234261.069025:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234261.069031:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234261.069034:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234261.069035:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069037:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234261.069038:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069040:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069042:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069044:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234261.069046:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234261.069047:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234261.069049:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234261.069050:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234261.069051:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069053:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234261.069054:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069055:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234261.069057:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234261.069164:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234261.069169:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234261.069174:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234261.708936:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234261.708942:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234261.708944:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234261.708949:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234261.708953:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234261.708957:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234261.708959:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234261.708961:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234261.708963:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234262.092970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234262.092974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234262.092979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234262.092979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234262.092981:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234262.092982:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234262.093037:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234262.093040:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234262.093057:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234262.732907:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234262.732911:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234262.732913:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234262.732917:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234262.732921:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234262.732924:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234262.732926:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234262.732928:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234262.732929:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234263.116951:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234263.116955:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234263.116961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234263.116962:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234263.116964:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234263.116966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234263.117039:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234263.117042:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234263.117044:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234263.756946:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234263.756951:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234263.756953:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234263.756956:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234263.756960:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234263.756962:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234263.756964:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234263.756966:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234263.756967:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234264.140958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234264.140963:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234264.140970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234264.140971:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234264.140974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234264.140975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234264.141045:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234264.141049:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234264.141052:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234264.780941:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234264.780947:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234264.780948:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234264.780953:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234264.780958:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234264.780962:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234264.780964:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234264.780966:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234264.780969:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234265.164975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234265.164978:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234265.165001:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234265.165002:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234265.165004:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234265.165005:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1761234265.165083:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1761234265.165087:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1761234265.165090:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234265.292967:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234265.292977:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234265.292978:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234265.292981:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234265.292997:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234265.293002:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234265.293008:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6734 00000100:00080000:0.0:1761234265.293010:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234265.293012:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293014:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234265.293016:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293017:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234265.293020:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234265.293021:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234265.293023:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234265.293026:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234265.293028:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234265.293029:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234265.293031:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293032:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234265.293033:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293034:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234265.293036:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234265.293038:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293039:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234265.293041:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234265.293043:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234265.293045:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234265.293046:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234265.293047:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234265.293055:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761234265.293057:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234265.293060:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234265.293061:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234265.293063:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234265.293066:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234265.293068:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234265.293076:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00000001:0.0:1761234265.293081:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234265.293086:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234265.293093:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234265.293095:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234265.293097:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234265.293111:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234265.293113:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234265.293114:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234265.293117:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234265.293118:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234265.293118:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234265.293119:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234265.293121:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234265.293122:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6734 00000100:00080000:0.0:1761234265.293123:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234265.293125:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293126:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234265.293127:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293128:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234265.293129:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234265.293130:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234265.293131:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234265.293132:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234265.293133:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234265.293134:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234265.293135:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293137:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234265.293139:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293140:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234265.293143:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234265.293145:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293147:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234265.293149:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1761234265.293153:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1761234265.293155:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234265.293157:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1761234265.293158:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293160:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1761234265.293162:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234265.293165:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000040:2.0:1761234265.293167:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000008:00000001:0.0:1761234265.293172:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234265.293175:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293179:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000010:0.0:1761234265.293182:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000100:00000001:2.0:1761234265.293182:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234265.293185:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1761234265.293186:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:2.0:1761234265.293188:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234265.293191:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00100000:2.0:1761234265.293192:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc434e0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235008:192.168.201.124@tcp:250: 02000000:00000001:0.0:1761234265.293193:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234265.293197:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234265.293199:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1761234265.293200:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234265.293203:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1761234265.293204:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234265.293206:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234265.293209:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 02000000:00000001:2.0:1761234265.293210:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1761234265.293216:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234265.293216:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234265.293219:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234265.293225:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:2.0:1761234265.293225:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234265.293232:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000001:0.0:1761234265.293235:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234265.293237:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234265.293239:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235008, portal 25 00000100:00000040:0.0:1761234265.293242:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293243:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234265.293245:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246323712 : -104717463227904 : ffffa0c290cd7200) 00000100:00000040:2.0:1761234265.293253:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234290 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234265.293257:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293261:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:0.0:1761234265.293262:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234265.293265:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1761234265.293266:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234265.293269:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000001:0.0:1761234265.293270:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00000200:2.0:1761234265.293272:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245235008, offset 0 00000100:00080000:0.0:1761234265.293273:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761234265.293276:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234265.293278:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1761234265.293281:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293283:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234265.293288:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6734 00000100:00080000:0.0:1761234265.293290:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234265.293292:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293294:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1761234265.293297:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000040:0.0:1761234265.293299:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293302:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234265.293305:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000400:00000200:2.0:1761234265.293306:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000020:00000040:0.0:1761234265.293307:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234265.293310:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000400:00000200:2.0:1761234265.293311:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1761234265.293313:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000400:00000200:2.0:1761234265.293316:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000020:00000001:0.0:1761234265.293318:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000200:2.0:1761234265.293322:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11889:11889:256:4294967295] 192.168.201.124@tcp LPNI seq info [11889:11889:8:4294967295] 00000020:00000001:0.0:1761234265.293323:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:0.0:1761234265.293330:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293335:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1761234265.293337:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000040:0.0:1761234265.293340:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293343:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000800:00000200:2.0:1761234265.293346:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000020:00000040:0.0:1761234265.293347:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000800:00000010:2.0:1761234265.293351:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000100:00080000:0.0:1761234265.293354:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234265.293358:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1761234265.293361:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000001:0.0:1761234265.293362:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234265.293365:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1761234265.293366:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 02000000:00000001:0.0:1761234265.293370:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761234265.293374:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:0.0:1761234265.293375:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234265.293377:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.1:1761234265.293378:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000008:00000001:0.0:1761234265.293384:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1761234265.293384:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234265.293387:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000100:00000001:0.0:1761234265.293388:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234265.293392:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000010:0.0:1761234265.293393:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000020:00000040:0.0:1761234265.293398:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000400:00000200:2.0:1761234265.293401:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1761234265.293404:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000400:00000200:2.0:1761234265.293406:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 02000000:00000001:0.0:1761234265.293407:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234265.293408:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:0.0:1761234265.293411:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234265.293412:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1761234265.293413:0:88993:0:(events.c:53:request_out_callback()) Process entered 02000000:00000001:0.0:1761234265.293414:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234265.293417:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234290 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 02000000:00000010:0.0:1761234265.293420:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761234265.293425:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293427:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00100000:0.0:1761234265.293434:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234265.293434:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234290 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293453:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234265.293457:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234265.293457:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:0.0:1761234265.293460:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293463:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293466:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1761234265.293467:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293468:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293472:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234265.293477:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234265/real 1761234265] req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234265.293482:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234265.293486:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000001:2.0:1761234265.293487:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234265.293489:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234265.293490:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000100:00000040:0.0:1761234265.293491:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6744) 00000400:00000200:2.0:1761234265.293492:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234265.293495:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234265.293497:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234265.293500:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234265.293504:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293508:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234265.293509:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293511:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234265.293512:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234265.293514:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293518:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234265.293522:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234265.293526:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234265.293527:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293528:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234265.293529:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293531:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234265.293532:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234265.293533:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234265.293535:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234265.293536:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234265.293538:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234265.293542:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc434e0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235008:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234265.293545:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293546:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234265.293547:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000dc434e0c x1846784245235008/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234265.293552:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234265.293553:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234265.293555:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234265.293557:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234265.293558:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234265.293559:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234265.293560:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234265.293562:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000007c64e188. 02000000:00000001:2.0:1761234265.293564:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234265.293565:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234265.293566:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000dc434e0c. 00000100:00000001:2.0:1761234265.293568:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234265.293569:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293570:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293576:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234265.293577:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293578:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234265.293579:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234265.293580:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234265.293582:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293586:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234265.293586:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234265.293588:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234265.293589:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234265.293590:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e728cb50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235072:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234265.293592:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234265.293593:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234265.293594:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234265.293595:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234265.293596:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234265.293598:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234265.293599:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234265.293601:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234265.293602:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235072, portal 10 00000100:00000001:2.0:1761234265.293604:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234265.293604:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246295744 : -104717463255872 : ffffa0c290cd04c0) 00000100:00000040:2.0:1761234265.293608:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234290 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293612:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234265.293614:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234265.293615:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234265.293616:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245235072, offset 0 00000400:00000200:2.0:1761234265.293618:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234265.293621:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234265.293623:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234265.293624:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234265.293626:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234265.293628:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11890:11890:256:4294967295] 192.168.201.124@tcp LPNI seq info [11890:11890:8:4294967295] 00000400:00000200:2.0:1761234265.293633:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234265.293636:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234265.293638:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234265.293640:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234265.293641:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761234265.293644:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234265.293645:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1761234265.293647:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234265.293649:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234265.293650:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234265.293652:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234265.293654:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234265.293655:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234265.293656:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234265.293657:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234290 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293662:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234265.293663:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234290 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293666:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293667:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234265.293669:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293670:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234265.293671:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293672:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234265.293673:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234265/real 1761234265] req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234265.293678:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234265.293679:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234265.293680:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234265.293681:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234265.293681:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234265.293682:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234265.293684:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234265.293688:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293691:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234265.293692:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293693:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234265.293694:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234265.293695:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293698:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234265.293700:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234265.293703:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234265.293704:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293705:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234265.293706:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293707:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234265.293707:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234265.293709:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234265.293710:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234265.293711:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234265.293713:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234265.293717:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e728cb50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235072:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234265.293719:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234265.293720:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293728:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234265.293729:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234265.293730:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234265.293731:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234265.293732:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a0dd9b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235136:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234265.293734:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234265.293735:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234265.293735:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234265.293736:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234265.293737:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234265.293738:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234265.293740:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234265.293741:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234265.293742:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235136, portal 4 00000100:00000001:2.0:1761234265.293743:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234265.293744:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246322496 : -104717463229120 : ffffa0c290cd6d40) 00000100:00000040:2.0:1761234265.293747:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234290 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293751:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234265.293752:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234265.293753:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234265.293754:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245235136, offset 0 00000400:00000200:2.0:1761234265.293756:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234265.293759:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234265.293761:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234265.293762:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234265.293763:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234265.293765:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11891:11891:256:4294967295] 192.168.201.124@tcp LPNI seq info [11891:11891:8:4294967295] 00000400:00000200:2.0:1761234265.293770:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234265.293772:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234265.293774:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234265.293776:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234265.293777:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1761234265.293779:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234265.293781:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1761234265.293783:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234265.293784:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234265.293785:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234265.293787:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234265.293788:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234265.293789:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234265.293790:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234265.293791:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234290 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293795:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234265.293796:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234290 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293800:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293801:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234265.293802:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293803:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234265.293804:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293805:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234265.293806:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234265/real 1761234265] req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234265.293810:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234265.293811:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234265.293811:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234265.293812:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234265.293813:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234265.293814:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234265.293815:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234265.293819:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293826:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234265.293826:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293827:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234265.293828:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234265.293830:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234265.293833:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234265.293834:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234265.293838:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234265.293839:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293840:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234265.293840:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293841:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234265.293842:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234265.293843:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234265.293867:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234265.293868:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234265.293870:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234265.293874:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a0dd9b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235136:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234265.293876:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293877:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234265.293878:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000e728cb50 x1846784245235072/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234265.293881:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234265.293882:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234265.293883:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234265.293884:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234265.293885:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234265.293886:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234265.293887:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234265.293888:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000005851ba37. 02000000:00000001:2.0:1761234265.293890:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234265.293891:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234265.293891:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000e728cb50. 00000100:00000001:2.0:1761234265.293893:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234265.293894:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293895:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234265.293896:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007a0dd9b5 x1846784245235136/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234290 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234265.293899:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234265.293900:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234265.293900:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234265.293902:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234265.293902:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234265.293903:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234265.293904:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234265.293904:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:2.0:1761234265.293905:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234265.293906:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234265.293906:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007a0dd9b5. 00000100:00000001:2.0:1761234265.293908:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234265.293908:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293909:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.293911:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234265.293912:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.293913:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234265.293914:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234265.804948:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234265.804952:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234265.804954:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234265.804958:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234265.804962:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234265.804966:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.804968:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234265.804971:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234265.804988:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234266.188940:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234266.188944:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234266.188950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234266.188951:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234266.188953:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234266.188954:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234266.189037:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234266.189040:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234266.189043:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234266.828938:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234266.828942:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234266.828943:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234266.828947:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234266.828950:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234266.828963:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234266.828965:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234266.828967:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234266.828969:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234267.212983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234267.212987:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234267.212993:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234267.212994:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234267.212997:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234267.212998:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234267.213035:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234267.213036:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234267.213038:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234267.852969:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234267.852974:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234267.852976:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234267.852980:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234267.852985:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234267.852988:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234267.852990:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234267.852992:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234267.852994:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234268.237003:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234268.237007:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234268.237013:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234268.237015:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234268.237017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234268.237018:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234268.237058:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234268.237061:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234268.237064:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234268.877013:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234268.877017:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234268.877018:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234268.877022:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234268.877026:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234268.877029:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234268.877030:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234268.877032:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234268.877034:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234269.260979:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234269.260984:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234269.260990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234269.260991:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234269.260993:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234269.260995:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234269.261013:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234269.261015:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234269.261017:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234269.901000:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234269.901005:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234269.901007:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234269.901010:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234269.901014:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234269.901017:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234269.901019:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234269.901020:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234269.901022:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234270.284961:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234270.284966:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234270.284972:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234270.284973:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234270.284975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234270.284976:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234270.285068:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234270.285071:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234270.285074:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234270.413081:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234270.413095:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234270.413096:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234270.413099:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234270.413102:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234270.413107:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234270.413115:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6739 00000100:00080000:3.0:1761234270.413118:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234270.413120:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413123:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234270.413125:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413126:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234270.413130:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234270.413131:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234270.413134:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234270.413138:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234270.413140:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234270.413142:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234270.413144:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413145:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234270.413146:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413147:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234270.413149:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234270.413151:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413153:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234270.413156:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234270.413159:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234270.413161:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413163:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234270.413165:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234270.413177:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a52d2576. 00000020:00000040:3.0:1761234270.413180:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234270.413183:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234270.413184:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413187:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234270.413190:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234270.413194:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234270.413203:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000296782d7. 00000100:00000001:3.0:1761234270.413210:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234270.413217:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234270.413225:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234270.413227:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234270.413230:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a52d2576] to pc [ptlrpcd_rcv+-1] req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234270.413252:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413254:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234270.413256:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234270.413259:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234270.413260:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234270.413261:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234270.413262:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234270.413263:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234270.413265:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6739 00000100:00080000:3.0:1761234270.413266:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234270.413267:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413268:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234270.413269:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413270:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234270.413271:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234270.413272:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234270.413273:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234270.413274:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234270.413275:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234270.413276:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234270.413278:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413279:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234270.413280:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413281:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234270.413283:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234270.413284:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413285:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234270.413286:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234270.413291:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234270.413292:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413293:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234270.413296:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234270.413299:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413301:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234270.413303:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000194cd3a7. 00000020:00000040:3.0:1761234270.413304:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761234270.413306:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234270.413307:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413307:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234270.413309:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234270.413310:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234270.413313:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000be06bfbf. 00000100:00000001:3.0:1761234270.413315:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234270.413319:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234270.413324:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234270.413325:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234270.413327:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000194cd3a7] to pc [ptlrpcd_rcv+-1] req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234270.413331:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413332:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234270.413333:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234270.413335:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234270.413336:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1761234270.413337:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234270.413338:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234270.413340:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:2.0:1761234270.413341:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1761234270.413343:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6739 00000100:00080000:3.0:1761234270.413346:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234270.413347:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234270.413348:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1761234270.413350:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1761234270.413354:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1761234270.413354:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413357:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234270.413360:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234270.413360:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761234270.413362:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234270.413363:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000040:2.0:1761234270.413367:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000020:00000001:3.0:1761234270.413367:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:3.0:1761234270.413375:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:3.0:1761234270.413387:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234270.413393:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234270.413399:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234270.413402:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1761234270.413403:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:2.0:1761234270.413405:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1761234270.413411:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 02000000:00000001:2.0:1761234270.413413:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1761234270.413416:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 02000000:00000001:2.0:1761234270.413417:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413418:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234270.413421:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1761234270.413422:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235200:192.168.201.124@tcp:250: 02000000:00000001:3.0:1761234270.413425:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234270.413429:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413434:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234270.413435:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1761234270.413441:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1761234270.413443:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234270.413448:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1761234270.413448:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234270.413455:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234270.413456:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1761234270.413460:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ecb9068. 00000020:00000040:3.0:1761234270.413462:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:3.0:1761234270.413464:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234270.413466:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234270.413468:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 00000100:00000001:3.0:1761234270.413475:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1761234270.413476:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234270.413481:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234270.413483:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234270.413485:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 02000000:00000010:3.0:1761234270.413486:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000062959873. 00000100:00000001:3.0:1761234270.413492:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234270.413495:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00100000:3.0:1761234270.413499:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234270.413504:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235200, portal 25 00000100:00000001:3.0:1761234270.413506:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234270.413507:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1761234270.413508:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.413510:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091112576 : -104717618439040 : ffffa0c2878d1c80) 00000100:00000040:3.0:1761234270.413515:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ecb9068] to pc [ptlrpcd_rcv+-1] req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234270.413521:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234295 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234270.413550:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.413552:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:3.0:1761234270.413556:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:2.0:1761234270.413559:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234270.413562:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000040:3.0:1761234270.413562:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6749) 00000100:00000200:2.0:1761234270.413568:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245235200, offset 0 00000400:00000200:2.0:1761234270.413576:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234270.413589:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234270.413595:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234270.413600:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234270.413603:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234270.413607:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11892:11892:256:4294967295] 192.168.201.124@tcp LPNI seq info [11892:11892:8:4294967295] 00000400:00000200:2.0:1761234270.413616:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234270.413620:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234270.413624:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234270.413630:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234270.413632:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761234270.413637:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234270.413640:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1761234270.413644:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234270.413645:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234270.413650:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234270.413654:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234270.413657:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234270.413659:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234270.413663:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234270.413665:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234295 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413672:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234270.413673:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234295 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413677:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.413679:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234270.413683:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.413684:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234270.413686:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.413688:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234270.413690:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234270/real 1761234270] req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234270.413695:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234270.413696:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234270.413697:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234270.413698:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234270.413699:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234270.413701:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234270.413702:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234270.413707:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413711:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234270.413712:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413713:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234270.413714:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234270.413716:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413720:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234270.413722:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234270.413726:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234270.413728:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413729:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234270.413729:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413731:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234270.413732:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234270.413734:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234270.413736:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234270.413738:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234270.413740:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234270.413744:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235200:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234270.413747:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234270.413749:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413753:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234270.413754:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234270.413755:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234270.413756:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234270.413758:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235264:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234270.413760:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234270.413761:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234270.413762:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234270.413763:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234270.413765:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234270.413766:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234270.413768:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234270.413769:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234270.413771:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235264, portal 10 00000100:00000001:2.0:1761234270.413772:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234270.413773:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091134464 : -104717618417152 : ffffa0c2878d7200) 00000100:00000040:2.0:1761234270.413777:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234295 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413780:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234270.413782:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234270.413783:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234270.413784:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245235264, offset 0 00000400:00000200:2.0:1761234270.413787:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234270.413790:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234270.413791:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234270.413793:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234270.413794:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234270.413795:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11893:11893:256:4294967295] 192.168.201.124@tcp LPNI seq info [11893:11893:8:4294967295] 00000400:00000200:2.0:1761234270.413800:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234270.413803:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234270.413804:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234270.413807:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234270.413808:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761234270.413810:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234270.413812:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1761234270.413814:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234270.413815:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234270.413817:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234270.413819:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234270.413820:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234270.413821:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234270.413822:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234270.413823:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234295 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413827:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234270.413828:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234295 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413831:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.413832:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234270.413834:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.413835:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234270.413836:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.413837:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234270.413838:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234270/real 1761234270] req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234270.413842:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234270.413842:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234270.413872:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234270.413874:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234270.413875:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234270.413876:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234270.413878:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234270.413882:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413903:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234270.413904:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413905:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234270.413906:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234270.413907:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413912:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234270.413913:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234270.413917:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234270.413918:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413919:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234270.413920:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413921:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234270.413922:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234270.413923:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234270.413924:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234270.413925:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234270.413927:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234270.413931:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235264:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234270.413934:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413935:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234270.413936:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a52d2576 x1846784245235200/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234270.413940:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234270.413942:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234270.413943:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234270.413946:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234270.413948:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234270.413949:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234270.413951:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234270.413953:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000296782d7. 02000000:00000001:2.0:1761234270.413956:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234270.413957:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234270.413959:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a52d2576. 00000100:00000001:2.0:1761234270.413962:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234270.413963:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413964:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234270.413965:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000194cd3a7 x1846784245235264/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234270.413968:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234270.413969:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234270.413970:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234270.413971:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234270.413972:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234270.413972:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234270.413973:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234270.413974:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000be06bfbf. 02000000:00000001:2.0:1761234270.413975:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234270.413976:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234270.413976:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000194cd3a7. 00000100:00000001:2.0:1761234270.413978:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234270.413978:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413980:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.413986:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234270.413987:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.413989:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234270.413990:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234270.413991:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234270.413993:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.413997:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234270.413997:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234270.413999:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234270.414000:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234270.414001:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235328:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234270.414003:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234270.414004:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234270.414005:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234270.414006:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234270.414008:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234270.414009:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234270.414011:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234270.414013:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234270.414014:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235328, portal 4 00000100:00000001:2.0:1761234270.414015:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234270.414016:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091118656 : -104717618432960 : ffffa0c2878d3440) 00000100:00000040:2.0:1761234270.414019:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234295 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.414023:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234270.414025:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234270.414026:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234270.414027:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245235328, offset 0 00000400:00000200:2.0:1761234270.414029:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234270.414032:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234270.414035:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234270.414036:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234270.414037:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234270.414039:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11894:11894:256:4294967295] 192.168.201.124@tcp LPNI seq info [11894:11894:8:4294967295] 00000400:00000200:2.0:1761234270.414044:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234270.414047:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234270.414048:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234270.414051:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234270.414052:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000200:2.1:1761234270.414054:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234270.414056:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 12 secs later) 00000800:00000100:2.0:1761234270.414058:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234270.414059:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234270.414061:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234270.414063:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234270.414064:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234270.414065:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234270.414067:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234270.414068:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234295 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.414072:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234270.414074:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234295 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.414077:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.414078:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234270.414080:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.414081:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234270.414082:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.414083:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234270.414084:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234270/real 1761234270] req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234270.414089:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234270.414090:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234270.414090:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234270.414091:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234270.414092:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234270.414093:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234270.414094:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234270.414098:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.414102:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234270.414102:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.414103:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234270.414104:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234270.414106:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234270.414109:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234270.414111:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234270.414118:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234270.414119:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.414120:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234270.414120:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.414121:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234270.414122:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234270.414123:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234270.414124:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234270.414125:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234270.414127:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234270.414131:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235328:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234270.414132:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.414133:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234270.414134:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000009ecb9068 x1846784245235328/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234295 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234270.414138:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234270.414138:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234270.414139:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234270.414140:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234270.414141:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234270.414141:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234270.414142:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234270.414143:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000062959873. 02000000:00000001:2.0:1761234270.414145:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234270.414145:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234270.414146:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000009ecb9068. 00000100:00000001:2.0:1761234270.414148:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234270.414149:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.414150:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.414151:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234270.414152:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.414153:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234270.414154:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234270.924949:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234270.924953:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234270.924954:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234270.924958:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234270.924963:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234270.924965:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.924967:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234270.924969:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234270.924971:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.308991:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234271.308997:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234271.308999:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234271.309001:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309006:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234271.309008:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309010:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309011:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309014:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234271.309016:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234271.309017:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234271.309020:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234271.309022:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234271.309022:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309023:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234271.309024:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309025:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309026:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309028:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234271.309029:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234271.309029:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234271.309031:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234271.309031:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234271.309032:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309033:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234271.309034:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309035:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234271.309037:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234271.309050:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234271.309052:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234271.309053:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234271.948977:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234271.948983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234271.948984:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234271.948987:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234271.948992:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234271.948995:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234271.948997:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234271.948999:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234271.949000:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234272.332978:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234272.332983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234272.332989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234272.332990:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234272.332993:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234272.332994:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234272.333052:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234272.333057:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234272.333061:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234272.972930:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234272.972936:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234272.972937:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234272.972942:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234272.972947:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234272.972950:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234272.972952:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234272.972953:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234272.972956:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234273.356970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234273.356974:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234273.356980:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234273.356981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234273.356985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234273.356987:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234273.357010:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234273.357012:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234273.357014:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234273.996976:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234273.996983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234273.996985:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234273.996989:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234273.996994:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234273.996998:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234273.997000:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234273.997003:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234273.997005:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234274.380936:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234274.380940:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234274.380945:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234274.380947:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234274.380950:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234274.380951:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234274.380976:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234274.380980:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234274.380983:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234275.020943:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234275.020947:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234275.020948:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234275.020952:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234275.020955:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234275.020958:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.020960:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234275.020962:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.020964:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234275.404930:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234275.404934:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234275.404939:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234275.404940:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234275.404942:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234275.404943:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234275.404964:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234275.404966:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234275.404969:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234275.532959:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234275.532970:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234275.532972:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234275.532975:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234275.532977:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234275.532984:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234275.532991:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6744 00000100:00080000:0.0:1761234275.532994:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234275.532996:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.532999:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234275.533000:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533002:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234275.533005:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234275.533007:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234275.533009:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234275.533012:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234275.533014:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234275.533015:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234275.533017:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533018:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234275.533019:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533021:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234275.533023:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234275.533026:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533027:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234275.533032:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234275.533034:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234275.533037:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533039:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234275.533040:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234275.533051:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761234275.533069:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234275.533074:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234275.533075:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533079:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234275.533082:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234275.533087:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234275.533096:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761234275.533103:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234275.533111:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234275.533120:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234275.533122:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234275.533125:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234275.533149:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533151:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234275.533152:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234275.533155:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234275.533156:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234275.533157:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234275.533158:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234275.533160:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234275.533162:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6744 00000100:00080000:0.0:1761234275.533163:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234275.533165:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533166:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234275.533167:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533169:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234275.533171:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234275.533172:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234275.533173:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234275.533174:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234275.533176:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234275.533177:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234275.533179:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533180:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234275.533181:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533182:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234275.533183:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234275.533184:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533185:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234275.533186:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234275.533192:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234275.533207:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533209:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234275.533212:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234275.533215:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533216:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234275.533219:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761234275.533220:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234275.533222:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234275.533223:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533224:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234275.533225:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234275.533226:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234275.533228:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761234275.533232:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234275.533235:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234275.533246:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234275.533247:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234275.533249:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533254:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1761234275.533255:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533258:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234275.533260:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234275.533260:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234275.533263:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234275.533265:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1761234275.533265:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1761234275.533267:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234275.533269:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234275.533272:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00000040:2.0:1761234275.533276:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00080000:0.0:1761234275.533277:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6744 00000100:00080000:0.0:1761234275.533281:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234275.533286:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533290:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234275.533293:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533297:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234275.533301:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234275.533302:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234275.533304:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234275.533305:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761234275.533309:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761234275.533313:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:2.0:1761234275.533314:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234275.533332:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234275.533333:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533337:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1761234275.533338:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1761234275.533339:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234275.533342:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 02000000:00000001:2.0:1761234275.533343:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1761234275.533344:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00100000:2.0:1761234275.533350:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2788073 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235392:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234275.533354:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234275.533359:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1761234275.533360:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 02000000:00000001:2.0:1761234275.533362:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533363:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234275.533368:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1761234275.533369:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1761234275.533370:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234275.533374:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533375:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 02000000:00000010:2.0:1761234275.533376:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 00000008:00100000:0.0:1761234275.533377:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1761234275.533381:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1761234275.533382:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533386:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234275.533391:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000400:00000010:2.0:1761234275.533391:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000020:00000040:0.0:1761234275.533397:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234275.533399:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000400:00000010:2.0:1761234275.533400:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 02000000:00000001:0.0:1761234275.533401:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234275.533405:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234275.533407:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234275.533408:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234275.533412:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235392, portal 25 02000000:00000010:0.0:1761234275.533413:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761234275.533419:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.533419:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234275.533422:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246317632 : -104717463233984 : ffffa0c290cd5a40) 00000100:00100000:0.0:1761234275.533426:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234275.533431:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234300 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234275.533444:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234275.533445:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.533448:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:0.0:1761234275.533452:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234275.533454:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234275.533459:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000001:0.0:1761234275.533463:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234275.533464:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245235392, offset 0 00000100:00000001:0.0:1761234275.533467:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761234275.533472:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6754) 00000400:00000200:2.0:1761234275.533475:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234275.533495:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234275.533505:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234275.533509:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234275.533512:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234275.533515:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11895:11895:256:4294967295] 192.168.201.124@tcp LPNI seq info [11895:11895:8:4294967295] 00000400:00000200:2.0:1761234275.533540:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234275.533546:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234275.533550:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234275.533558:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234275.533561:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761234275.533566:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234275.533569:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1761234275.533572:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234275.533574:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234275.533579:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234275.533583:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234275.533587:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234275.533589:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234275.533594:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234275.533596:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234300 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533603:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234275.533605:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234300 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533610:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.533611:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234275.533616:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.533618:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234275.533620:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.533621:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234275.533624:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234275/real 1761234275] req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234275.533631:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234275.533633:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234275.533634:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234275.533635:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234275.533636:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234275.533637:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234275.533639:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234275.533644:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533648:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234275.533649:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533651:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234275.533653:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234275.533655:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533659:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234275.533662:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234275.533665:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234275.533667:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533668:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234275.533669:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533670:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234275.533672:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234275.533673:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234275.533675:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234275.533676:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234275.533678:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234275.533683:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2788073 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235392:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234275.533686:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234275.533687:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533691:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234275.533692:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234275.533693:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234275.533695:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234275.533696:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086a5741d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235456:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234275.533697:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234275.533698:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234275.533700:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234275.533701:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234275.533703:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234275.533704:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234275.533706:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234275.533708:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234275.533710:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235456, portal 10 00000100:00000001:2.0:1761234275.533711:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234275.533712:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246304256 : -104717463247360 : ffffa0c290cd2600) 00000100:00000040:2.0:1761234275.533716:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234300 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533720:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234275.533722:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234275.533723:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234275.533724:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245235456, offset 0 00000400:00000200:2.0:1761234275.533727:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234275.533729:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234275.533731:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234275.533733:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234275.533734:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234275.533735:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11896:11896:256:4294967295] 192.168.201.124@tcp LPNI seq info [11896:11896:8:4294967295] 00000400:00000200:2.0:1761234275.533740:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234275.533743:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234275.533745:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234275.533747:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234275.533748:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761234275.533751:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234275.533752:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1761234275.533755:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234275.533756:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234275.533757:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234275.533759:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234275.533761:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234275.533762:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234275.533763:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234275.533765:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234300 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533769:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234275.533770:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234300 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533774:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.533775:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234275.533776:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.533777:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234275.533779:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.533780:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234275.533781:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234275/real 1761234275] req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234275.533785:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234275.533785:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234275.533786:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234275.533787:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234275.533805:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234275.533807:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234275.533809:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234275.533813:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533817:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234275.533818:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533819:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234275.533820:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234275.533821:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533825:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234275.533826:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234275.533831:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234275.533833:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533833:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234275.533834:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533835:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234275.533836:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234275.533837:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234275.533838:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234275.533839:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234275.533841:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234275.533872:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a5741d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235456:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234275.533874:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533876:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234275.533877:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000c2788073 x1846784245235392/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234275.533882:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234275.533885:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234275.533886:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234275.533889:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234275.533891:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234275.533892:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234275.533893:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234275.533895:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078138607. 02000000:00000001:2.0:1761234275.533899:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234275.533900:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234275.533901:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000c2788073. 00000100:00000001:2.0:1761234275.533903:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234275.533904:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533905:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234275.533907:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000086a5741d x1846784245235456/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234275.533911:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234275.533912:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234275.533913:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234275.533914:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234275.533915:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234275.533915:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234275.533916:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234275.533917:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:2.0:1761234275.533918:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234275.533919:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234275.533919:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000086a5741d. 00000100:00000001:2.0:1761234275.533920:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234275.533921:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533922:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.533930:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234275.533931:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.533932:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234275.533933:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234275.533934:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234275.533935:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533939:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234275.533939:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234275.533940:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234275.533941:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234275.533943:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f5a405a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235520:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234275.533944:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234275.533945:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234275.533946:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234275.533948:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234275.533950:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234275.533951:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234275.533953:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234275.533955:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234275.533957:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235520, portal 4 00000100:00000001:2.0:1761234275.533958:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234275.533959:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246296960 : -104717463254656 : ffffa0c290cd0980) 00000100:00000040:2.0:1761234275.533962:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234300 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.533967:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234275.533969:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234275.533971:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234275.533973:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245235520, offset 0 00000400:00000200:2.0:1761234275.533978:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234275.533982:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234275.533986:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234275.533988:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234275.533990:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234275.533992:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11897:11897:256:4294967295] 192.168.201.124@tcp LPNI seq info [11897:11897:8:4294967295] 00000400:00000200:2.0:1761234275.533999:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234275.534004:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234275.534007:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234275.534011:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234275.534013:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000200:2.1:1761234275.534017:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234275.534019:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 7 secs later) 00000800:00000100:2.0:1761234275.534039:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234275.534041:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234275.534044:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234275.534048:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234275.534051:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234275.534052:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234275.534055:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234275.534058:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234300 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.534066:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234275.534069:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234300 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.534076:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.534078:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234275.534081:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.534083:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234275.534084:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.534086:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234275.534089:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234275/real 1761234275] req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234275.534098:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234275.534099:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234275.534101:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234275.534103:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234275.534104:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234275.534106:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234275.534109:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234275.534117:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.534124:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234275.534126:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.534127:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234275.534129:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234275.534132:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234275.534139:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234275.534142:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234275.534149:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234275.534151:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.534152:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234275.534154:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.534155:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234275.534156:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234275.534158:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234275.534160:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234275.534161:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234275.534163:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234275.534168:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f5a405a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235520:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234275.534171:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.534172:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234275.534173:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000f5a405a x1846784245235520/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234300 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234275.534177:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234275.534178:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234275.534179:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234275.534181:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234275.534182:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234275.534183:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234275.534185:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234275.534185:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000005851ba37. 02000000:00000001:2.0:1761234275.534187:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234275.534188:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234275.534189:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000f5a405a. 00000100:00000001:2.0:1761234275.534191:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234275.534192:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.534193:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234275.534196:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234275.534197:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234275.534198:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234275.534200:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234276.044936:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234276.044942:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234276.044945:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234276.044950:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234276.044955:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234276.044959:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234276.044961:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234276.044964:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234276.044966:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234276.428989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234276.428995:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234276.429001:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234276.429002:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234276.429005:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234276.429006:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234276.429088:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234276.429092:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234276.429096:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234277.068977:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234277.068983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234277.068985:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234277.068990:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234277.068996:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234277.069000:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234277.069002:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234277.069005:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234277.069007:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234277.453028:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234277.453035:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234277.453045:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234277.453046:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234277.453051:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234277.453052:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234277.453077:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234277.453080:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234277.453083:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234278.092990:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234278.092998:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234278.093000:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234278.093006:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234278.093012:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234278.093017:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234278.093020:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234278.093023:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234278.093026:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234278.477009:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234278.477017:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234278.477026:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234278.477027:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234278.477031:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234278.477032:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234278.477098:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234278.477102:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234278.477106:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234279.117008:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234279.117014:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234279.117016:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234279.117021:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234279.117028:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234279.117034:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234279.117037:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234279.117040:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234279.117043:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234279.501011:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234279.501017:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234279.501026:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234279.501027:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234279.501031:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234279.501032:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234279.501061:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234279.501064:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234279.501067:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234280.140952:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234280.140958:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234280.140960:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234280.140966:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234280.140971:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234280.140977:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.140980:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234280.140983:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.140985:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234280.525002:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234280.525008:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234280.525015:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234280.525017:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234280.525020:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234280.525021:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234280.525124:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234280.525130:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234280.525135:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234280.653017:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234280.653029:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234280.653030:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234280.653033:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234280.653036:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234280.653042:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234280.653051:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6749 00000100:00080000:0.0:1761234280.653056:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234280.653059:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653063:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234280.653065:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653067:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234280.653071:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234280.653073:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234280.653075:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234280.653079:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234280.653080:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234280.653083:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234280.653085:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653087:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234280.653091:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653093:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234280.653096:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234280.653099:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653101:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234280.653104:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234280.653107:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234280.653109:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234280.653111:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234280.653114:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234280.653127:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000095969832. 00000020:00000040:0.0:1761234280.653131:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234280.653135:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234280.653136:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234280.653139:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234280.653142:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234280.653146:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234280.653155:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761234280.653161:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234280.653168:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234280.653177:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234280.653179:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234280.653182:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000095969832] to pc [ptlrpcd_rcv+-1] req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234280.653197:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234280.653199:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234280.653200:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234280.653202:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234280.653202:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234280.653203:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234280.653204:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234280.653206:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234280.653208:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6749 00000100:00080000:0.0:1761234280.653209:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234280.653211:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653213:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234280.653214:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653216:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234280.653217:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234280.653218:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234280.653219:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234280.653220:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234280.653221:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234280.653222:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234280.653224:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234280.653225:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1761234280.653226:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234280.653228:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653230:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:2.0:1761234280.653231:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1761234280.653232:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234280.653235:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:2.0:1761234280.653235:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1761234280.653239:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234280.653240:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1761234280.653242:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:0.0:1761234280.653246:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234280.653249:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234280.653252:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234280.653254:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1761234280.653256:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1761234280.653257:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1761234280.653261:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234280.653262:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1761234280.653263:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234280.653266:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234280.653267:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000002ca01b78. 00000100:00100000:2.0:1761234280.653270:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000095969832 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235584:192.168.201.124@tcp:250: 00000020:00000040:0.0:1761234280.653271:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234280.653274:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234280.653276:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653276:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1761234280.653278:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234280.653280:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234280.653282:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234280.653282:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234280.653285:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234280.653286:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 02000000:00000001:2.0:1761234280.653289:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1761234280.653291:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1761234280.653295:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 00000100:00100000:0.0:1761234280.653296:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234280.653298:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234280.653304:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234280.653305:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234280.653306:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00000040:0.0:1761234280.653308:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000002ca01b78] to pc [ptlrpcd_rcv+-1] req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234280.653315:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234280.653315:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000001:0.0:1761234280.653318:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234280.653319:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234280.653321:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234280.653322:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761234280.653322:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234280.653324:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1761234280.653325:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235584, portal 25 00000100:00000001:0.0:1761234280.653326:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234280.653329:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6749 00000100:00000001:2.0:1761234280.653329:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1761234280.653331:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1761234280.653332:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246303040 : -104717463248576 : ffffa0c290cd2140) 00000100:00000040:0.0:1761234280.653335:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653337:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1761234280.653339:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234305 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:0.0:1761234280.653341:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653344:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234280.653348:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234280.653349:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234280.653351:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234280.653353:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000001:2.0:1761234280.653353:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000001:0.0:1761234280.653356:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234280.653359:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000020:00000001:0.0:1761234280.653360:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000010:2.0:1761234280.653362:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000040:0.0:1761234280.653364:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000200:2.0:1761234280.653366:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245235584, offset 0 00000100:00000001:0.0:1761234280.653367:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234280.653370:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653371:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234280.653372:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00080000:0.0:1761234280.653376:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000200:2.0:1761234280.653377:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653379:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234280.653381:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234280.653382:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234280.653384:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234280.653385:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234280.653386:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234280.653387:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234280.653388:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234280.653392:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000f5a405a. 00000020:00000040:0.0:1761234280.653394:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234280.653395:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234280.653397:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234280.653397:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000001:0.0:1761234280.653398:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234280.653401:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234280.653402:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234280.653404:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761234280.653407:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234280.653407:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00100000:0.0:1761234280.653411:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234280.653414:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234280.653420:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000001:0.0:1761234280.653422:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234280.653423:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234280.653426:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11898:11898:256:4294967295] 192.168.201.124@tcp LPNI seq info [11898:11898:8:4294967295] 00000100:00000040:0.0:1761234280.653429:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000f5a405a] to pc [ptlrpcd_rcv+-1] req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234280.653435:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234280.653437:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000200:2.0:1761234280.653440:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000040:0.0:1761234280.653441:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6759) 00000800:00000200:2.0:1761234280.653448:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234280.653453:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234280.653460:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234280.653463:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761234280.653468:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234280.653471:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1761234280.653474:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234280.653477:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234280.653483:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234280.653488:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234280.653492:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234280.653493:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234280.653498:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234280.653501:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234305 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653507:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234280.653509:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234305 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653514:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653515:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234280.653520:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653521:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234280.653523:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653525:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234280.653527:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234280/real 1761234280] req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234280.653533:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234280.653535:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234280.653536:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234280.653537:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234280.653539:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234280.653540:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234280.653541:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234280.653547:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653550:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234280.653551:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653553:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234280.653554:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234280.653556:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653560:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234280.653562:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234280.653566:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234280.653568:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653569:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234280.653570:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653572:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234280.653573:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234280.653575:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234280.653576:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234280.653578:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234280.653580:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234280.653585:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000095969832 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235584:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234280.653589:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653590:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234280.653591:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000095969832 x1846784245235584/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234280.653596:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234280.653598:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234280.653600:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234280.653602:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234280.653604:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234280.653605:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234280.653606:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234280.653608:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:2.0:1761234280.653612:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234280.653613:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234280.653614:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000095969832. 00000100:00000001:2.0:1761234280.653616:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234280.653617:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653618:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653626:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234280.653627:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653628:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234280.653630:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234280.653630:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234280.653632:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653636:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234280.653637:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234280.653638:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234280.653640:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234280.653641:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000002ca01b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235648:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234280.653643:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234280.653644:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234280.653645:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234280.653646:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234280.653647:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234280.653648:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234280.653651:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234280.653652:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234280.653654:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235648, portal 10 00000100:00000001:2.0:1761234280.653656:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234280.653657:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246321280 : -104717463230336 : ffffa0c290cd6880) 00000100:00000040:2.0:1761234280.653661:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234305 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653665:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234280.653666:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234280.653667:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234280.653669:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245235648, offset 0 00000400:00000200:2.0:1761234280.653671:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234280.653674:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234280.653676:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234280.653678:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234280.653679:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234280.653681:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11899:11899:256:4294967295] 192.168.201.124@tcp LPNI seq info [11899:11899:8:4294967295] 00000400:00000200:2.0:1761234280.653687:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234280.653689:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234280.653707:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234280.653710:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234280.653712:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761234280.653715:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234280.653716:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1761234280.653718:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234280.653720:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234280.653722:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234280.653724:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234280.653726:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234280.653727:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234280.653728:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234280.653730:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234305 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653735:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234280.653737:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234305 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653744:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653745:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234280.653747:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653748:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234280.653749:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653750:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234280.653752:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234280/real 1761234280] req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234280.653757:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234280.653758:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234280.653759:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234280.653760:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234280.653761:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234280.653762:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234280.653763:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234280.653767:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653770:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234280.653771:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653772:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234280.653773:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234280.653774:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653777:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234280.653779:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234280.653783:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234280.653784:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653785:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234280.653786:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653787:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234280.653789:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234280.653790:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234280.653792:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234280.653793:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234280.653795:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234280.653799:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000002ca01b78 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235648:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234280.653801:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234280.653802:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653805:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234280.653806:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234280.653807:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234280.653808:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234280.653809:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000f5a405a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235712:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234280.653811:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234280.653812:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234280.653813:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234280.653814:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234280.653817:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234280.653817:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234280.653819:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234280.653821:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234280.653823:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235712, portal 4 00000100:00000001:2.0:1761234280.653824:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234280.653825:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246296960 : -104717463254656 : ffffa0c290cd0980) 00000100:00000040:2.0:1761234280.653828:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234305 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653832:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234280.653834:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234280.653835:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000100:00000200:2.0:1761234280.653837:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245235712, offset 0 00000400:00000200:2.0:1761234280.653839:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234280.653842:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234280.653868:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234280.653870:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234280.653871:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234280.653873:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11900:11900:256:4294967295] 192.168.201.124@tcp LPNI seq info [11900:11900:8:4294967295] 00000400:00000200:2.0:1761234280.653878:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234280.653881:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234280.653884:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 00000800:00000200:2.0:1761234280.653886:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234280.653888:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1761234280.653890:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234280.653891:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1761234280.653894:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234280.653895:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676282). 00000400:00000200:2.0:1761234280.653897:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234280.653899:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234280.653901:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:2.0:1761234280.653902:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:2.0:1761234280.653903:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234280.653905:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234305 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653910:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234280.653911:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234305 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653915:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653916:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234280.653918:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653919:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234280.653920:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.653922:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234280.653923:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234280/real 1761234280] req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234280.653928:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234280.653929:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234280.653929:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234280.653930:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234280.653931:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234280.653932:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234280.653933:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234280.653937:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653941:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234280.653942:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653943:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234280.653943:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234280.653945:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234280.653948:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234280.653950:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234280.653953:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234280.653954:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653955:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234280.653956:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653957:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234280.653957:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234280.653959:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234280.653960:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234280.653961:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234280.653964:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234280.653968:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000f5a405a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235712:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234280.653971:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653972:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234280.653973:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000002ca01b78 x1846784245235648/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234280.653976:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234280.653977:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234280.653978:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234280.653979:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234280.653980:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234280.653980:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234280.653981:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234280.653982:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078138607. 02000000:00000001:2.0:1761234280.653983:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234280.653984:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234280.653985:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000002ca01b78. 00000100:00000001:2.0:1761234280.653987:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234280.653987:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.653988:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234280.653990:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000f5a405a x1846784245235712/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234305 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234280.653993:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234280.653994:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234280.653994:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234280.653996:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234280.653996:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234280.653997:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234280.653998:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234280.653998:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000040aa0f54. 02000000:00000001:2.0:1761234280.653999:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234280.654000:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234280.654001:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000f5a405a. 00000100:00000001:2.0:1761234280.654003:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234280.654003:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.654004:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234280.654007:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234280.654007:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234280.654008:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234280.654009:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234281.164986:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234281.164992:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234281.164993:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234281.164998:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234281.165003:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234281.165007:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234281.165009:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234281.165012:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234281.165014:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.548971:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234281.548978:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234281.548981:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234281.548983:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.548990:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234281.548992:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.548994:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.548996:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.548999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234281.549000:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234281.549001:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234281.549006:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234281.549008:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234281.549009:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.549010:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234281.549011:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.549012:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.549013:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.549016:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234281.549017:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234281.549017:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234281.549019:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234281.549021:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234281.549022:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.549023:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234281.549024:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.549025:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234281.549026:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234281.549045:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234281.549047:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234281.549049:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234282.188934:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234282.188943:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234282.188946:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234282.188952:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234282.188959:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234282.188963:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234282.188967:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234282.188970:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234282.188972:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234282.572980:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234282.572987:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234282.572993:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234282.572994:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234282.572998:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234282.572998:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234282.573087:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234282.573091:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234282.573095:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234283.213030:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234283.213037:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234283.213039:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234283.213045:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234283.213051:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234283.213055:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234283.213058:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234283.213061:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234283.213063:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234283.596989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234283.596996:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234283.597004:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234283.597005:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234283.597011:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234283.597015:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234283.597042:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234283.597044:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234283.597047:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234284.237028:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234284.237037:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234284.237040:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234284.237047:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234284.237054:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234284.237060:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234284.237063:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234284.237077:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234284.237080:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234284.621007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234284.621015:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234284.621023:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234284.621025:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234284.621032:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234284.621034:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234284.621139:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234284.621143:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234284.621148:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234285.261013:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234285.261024:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234285.261026:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234285.261033:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234285.261040:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234285.261045:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234285.261048:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234285.261051:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234285.261054:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234285.645009:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234285.645015:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234285.645024:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234285.645026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234285.645029:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234285.645030:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234285.645055:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234285.645057:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234285.645060:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234285.773023:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234285.773032:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234285.773033:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234285.773036:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234285.773037:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234285.773042:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234285.773047:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6754 00000100:00080000:0.0:1761234285.773049:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234285.773052:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773054:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234285.773056:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773058:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234285.773060:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234285.773062:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234285.773064:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234285.773066:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234285.773067:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234285.773070:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234285.773072:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773074:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234285.773076:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773078:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234285.773080:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234285.773082:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773084:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234285.773087:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234285.773089:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234285.773091:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773093:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234285.773095:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234285.773102:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000086a5741d. 00000020:00000040:0.0:1761234285.773105:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234285.773108:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234285.773109:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773111:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234285.773113:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234285.773116:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234285.773122:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761234285.773126:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234285.773132:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234285.773138:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234285.773140:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234285.773142:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000086a5741d] to pc [ptlrpcd_rcv+-1] req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234285.773156:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773157:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234285.773158:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234285.773160:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234285.773161:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234285.773162:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234285.773163:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234285.773165:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234285.773166:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6754 00000100:00080000:0.0:1761234285.773168:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234285.773169:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773170:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234285.773171:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773172:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234285.773173:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234285.773173:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234285.773174:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234285.773175:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234285.773176:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234285.773177:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234285.773179:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773180:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234285.773181:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773182:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234285.773183:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234285.773185:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773186:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234285.773187:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234285.773192:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234285.773193:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773195:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234285.773197:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234285.773200:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773202:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234285.773205:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000c2788073. 00000020:00000040:0.0:1761234285.773207:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234285.773210:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234285.773211:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773212:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234285.773214:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234285.773215:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773218:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1761234285.773219:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761234285.773228:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773229:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234285.773233:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1761234285.773238:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000040:2.0:1761234285.773245:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234285.773268:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234285.773268:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234285.773270:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773272:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234285.773276:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1761234285.773281:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000c2788073] to pc [ptlrpcd_rcv+-1] req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234285.773282:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234285.773289:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000086a5741d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235776:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234285.773298:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234285.773303:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234285.773307:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773308:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234285.773312:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1761234285.773313:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234285.773317:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234285.773321:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 02000000:00000010:2.0:1761234285.773321:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 00000100:00080000:0.0:1761234285.773324:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1761234285.773325:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773329:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234285.773331:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1761234285.773333:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00000001:0.0:1761234285.773335:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234285.773342:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6754 00000400:00000010:2.0:1761234285.773342:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00080000:0.0:1761234285.773349:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1761234285.773352:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235776, portal 25 00000100:00000040:0.0:1761234285.773354:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234285.773357:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234285.773361:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246304256 : -104717463247360 : ffffa0c290cd2600) 00000100:00000001:0.0:1761234285.773362:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234285.773369:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000040:2.0:1761234285.773371:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234310 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234285.773373:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234285.773381:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234285.773383:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234285.773389:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000100:00000001:2.0:1761234285.773389:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000040:0.0:1761234285.773395:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000040:2.0:1761234285.773396:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234285.773400:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000411dce81. 00000020:00000001:0.0:1761234285.773402:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000200:2.0:1761234285.773404:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245235776, offset 0 00000020:00000001:0.0:1761234285.773411:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000200:2.0:1761234285.773418:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000040:0.0:1761234285.773421:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773425:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234285.773427:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773429:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234285.773432:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000200:2.0:1761234285.773437:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00080000:0.0:1761234285.773438:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234285.773443:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234285.773446:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 02000000:00000001:0.0:1761234285.773447:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1761234285.773452:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1761234285.773453:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1761234285.773457:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 02000000:00000001:0.0:1761234285.773460:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234285.773464:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11901:11901:256:4294967295] 192.168.201.124@tcp LPNI seq info [11901:11901:8:4294967295] 00000100:00000001:0.0:1761234285.773466:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234285.773471:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234285.773476:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234285.773481:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:0.0:1761234285.773482:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234285.773491:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000007a0dd9b5. 00000800:00000200:2.0:1761234285.773491:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000020:00000040:0.0:1761234285.773500:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000800:00000010:2.0:1761234285.773502:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000dbf45c28 (tot 1676530). 02000000:00000001:0.0:1761234285.773505:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234285.773507:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773511:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:2.0:1761234285.773513:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000001:0.0:1761234285.773514:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234285.773521:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1761234285.773522:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 02000000:00000010:0.0:1761234285.773532:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761234285.773538:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234285.773544:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234285.773549:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234285.773552:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234285.773553:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1761234285.773555:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773556:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773563:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773565:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1761234285.773568:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000007a0dd9b5] to pc [ptlrpcd_rcv+-1] req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234285.773576:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1761234285.773577:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773579:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1761234285.773585:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1761234285.773586:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000001:2.0:1761234285.773591:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234285.773593:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1761234285.773607:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6764) 00000100:00000040:2.0:1761234285.773624:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234285.773630:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234285.773633:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234285.773636:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234285.773638:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234285.773641:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000c2788073 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235840:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234285.773643:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234285.773645:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234285.773646:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234285.773647:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234285.773650:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234285.773652:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234285.773656:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000003966d2af. 00000400:00000010:2.0:1761234285.773659:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000001d64cf84. 00000100:00000200:2.0:1761234285.773661:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235840, portal 10 00000100:00000001:2.0:1761234285.773662:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234285.773663:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246317632 : -104717463233984 : ffffa0c290cd5a40) 00000100:00000040:2.0:1761234285.773670:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234310 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000400:00000200:3.0:1761234285.773670:0:88984:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.124:988 00000100:00000001:2.0:1761234285.773688:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234285.773691:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234285.773693:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000c236fb48. 00000100:00000200:2.0:1761234285.773695:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245235840, offset 0 00000400:00000200:2.0:1761234285.773699:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234285.773706:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234285.773708:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234285.773710:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234285.773712:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234285.773716:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11902:11902:255:4294967295] 192.168.201.124@tcp LPNI seq info [11902:11902:7:4294967295] 00000400:00000200:2.0:1761234285.773725:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234285.773731:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234285.773734:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000007ceb0618 (tot 1676778). 00000800:00000200:2.0:1761234285.773739:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234285.773742:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:2.0:1761234285.773744:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773745:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234285.773747:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773750:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234285.773755:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234285.773760:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234285.773762:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234285.773765:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234285.773769:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1761234285.773770:0:88984:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.124:988 00000100:00100000:2.0:1761234285.773777:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000007a0dd9b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235904:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234285.773782:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234285.773784:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234285.773786:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234285.773788:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234285.773791:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234285.773794:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234285.773800:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 00000000ccddd6a4. 00000400:00000010:2.0:1761234285.773805:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234285.773807:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235904, portal 4 00000100:00000001:2.0:1761234285.773809:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234285.773811:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246322496 : -104717463229120 : ffffa0c290cd6d40) 00000400:00000200:3.0:1761234285.773816:0:88984:0:(lib-socket.c:430:lnet_sock_connect()) Error -99 connecting 1021 -> 192.168.201.124:988 00000100:00000040:2.0:1761234285.773818:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234310 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234285.773827:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234285.773832:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234285.773835:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 000000003e4cc038. 00000100:00000200:2.0:1761234285.773839:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245235904, offset 0 00000400:00000200:2.0:1761234285.773908:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234285.773914:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234285.773917:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234285.773920:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234285.773923:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234285.773927:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11903:11903:254:4294967295] 192.168.201.124@tcp LPNI seq info [11903:11903:6:4294967295] 00000400:00000200:2.0:1761234285.773942:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234285.773950:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234285.773956:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009b62b3c1 (tot 1677026). 00000800:00000200:2.0:1761234285.773972:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234285.773980:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:2.0:1761234285.773983:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773985:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234285.773986:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773989:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.773990:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234285.773994:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234285.773995:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1761234285.773997:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234285.773998:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234285.773999:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234285.774000:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234286.285030:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234286.285047:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234286.285050:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234286.285057:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234286.285064:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234286.285068:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234286.285071:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234286.285075:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234286.285078:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234286.669001:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234286.669008:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234286.669014:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234286.669015:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234286.669017:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234286.669018:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234286.669083:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234286.669087:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234286.669090:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234287.309001:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234287.309008:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234287.309010:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234287.309014:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234287.309021:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234287.309025:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234287.309027:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234287.309030:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234287.309033:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234287.692952:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234287.692959:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234287.692968:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234287.692969:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234287.692974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234287.692975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1761234287.692998:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1761234287.693001:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1761234287.693003:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234288.332997:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234288.333003:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234288.333005:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234288.333011:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234288.333017:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234288.333020:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.333023:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234288.333026:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.333028:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234288.717019:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234288.717026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234288.717032:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234288.717033:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234288.717036:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234288.717037:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234288.717104:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234288.717108:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234288.717112:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1761234288.845108:0:88984:0:(lib-socket.c:430:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.201.124:988 00000400:00000100:3.0:1761234288.845153:0:88984:0:(acceptor.c:105:lnet_connect_console_error()) Connection to 192.168.201.124@tcp at host 192.168.201.124 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1761234288.845162:0:88984:0:(router.c:1753:lnet_notify()) 192.168.201.24@tcp notifying 192.168.201.124@tcp: down 00000800:00000100:3.0:1761234288.845172:0:88984:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:3.0:1761234288.845175:0:88984:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000dbf45c28 (tot 1676778). 00000400:00000200:3.0:1761234288.845181:0:88984:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1761234288.845186:0:88984:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1761234288.845192:0:88984:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000411dce81 00000400:00000010:3.0:1761234288.845194:0:88984:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000411dce81. 00000100:00000001:3.0:1761234288.845199:0:88984:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234288.845205:0:88984:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234310 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234288.845226:0:88984:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234288.845230:0:88984:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234310 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234288.845236:0:88984:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234288.845239:0:88984:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:3.0:1761234288.845247:0:88984:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:3.0:1761234288.845249:0:88984:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000007ceb0618 (tot 1676530). 00000400:00000200:3.0:1761234288.845252:0:88984:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1761234288.845255:0:88984:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1761234288.845257:0:88984:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000c236fb48 00000400:00000010:3.0:1761234288.845259:0:88984:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000c236fb48. 00000100:00000001:3.0:1761234288.845261:0:88984:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:3.0:1761234288.845264:0:88984:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234310 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234288.845271:0:88984:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:3.0:1761234288.845274:0:88984:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234310 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234288.845283:0:88984:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234288.845286:0:88984:0:(events.c:93:request_out_callback()) Process leaving 00000800:00000100:3.0:1761234288.845291:0:88984:0:(socklnd_cb.c:430:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.124@tcp 00000800:00000010:3.0:1761234288.845294:0:88984:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009b62b3c1 (tot 1676282). 00000400:00000200:3.0:1761234288.845298:0:88984:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:2.0:1761234288.845302:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1761234288.845304:0:88984:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_DROPPED 00000100:00000001:2.0:1761234288.845309:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1761234288.845310:0:88984:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000003e4cc038 00000100:00000001:2.0:1761234288.845313:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000400:00000010:3.0:1761234288.845315:0:88984:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000003e4cc038. 00000100:00000001:3.0:1761234288.845322:0:88984:0:(events.c:53:request_out_callback()) Process entered 00000100:00100000:2.0:1761234288.845327:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234285/real 1761234288] req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:3.0:1761234288.845331:0:88984:0:(events.c:64:request_out_callback()) @@@ type 5, status -113 req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234310 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234288.845364:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1761234288.845365:0:88984:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000400:00000001:2.0:1761234288.845367:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234288.845372:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000003966d2af. 00000400:00000200:2.0:1761234288.845378:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 000000001d64cf84 00000400:00000010:2.0:1761234288.845380:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 000000001d64cf84. 00000100:00000001:2.0:1761234288.845384:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000040:3.0:1761234288.845385:0:88984:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234310 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234288.845389:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234288.845438:0:88984:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234288.845439:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234288.845445:0:88984:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234288.845454:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234288.845457:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845461:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234288.845463:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234288.845471:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234288.845478:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234288.845482:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234288.845489:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234288.845494:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845495:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234288.845496:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845499:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234288.845502:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234288.845506:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234288.845512:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234288.845514:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234288.845518:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234288.845526:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000c2788073 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235840:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234288.845532:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234288.845578:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234285/real 1761234288] req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234288.845587:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234288.845588:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234288.845590:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 00000000ccddd6a4. 00000400:00000200:2.0:1761234288.845593:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234288.845594:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234288.845597:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234288.845599:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234288.845605:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234288.845610:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234288.845612:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845613:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234288.845614:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234288.845617:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234288.845629:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234288.845632:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234288.845638:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234288.845640:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845641:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234288.845642:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845644:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234288.845646:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234288.845648:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234288.845650:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234288.845653:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234288.845656:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234288.845662:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000007a0dd9b5 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245235904:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234288.845667:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234288.845670:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234285/real 1761234288] req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234288.845677:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234288.845679:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234288.845680:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234288.845682:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234288.845684:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234288.845686:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234288.845688:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234288.845694:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234288.845705:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234288.845706:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845708:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234288.845709:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234288.845712:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234288.845718:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234288.845720:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234288.845726:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234288.845728:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845729:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234288.845730:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845732:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234288.845733:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234288.845735:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234288.845739:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234288.845741:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234288.845744:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234288.845750:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000086a5741d pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235776:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234288.845755:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845758:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234288.845761:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000c2788073 x1846784245235840/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234288.845768:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234288.845772:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234288.845777:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234288.845783:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234288.845786:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234288.845788:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234288.845792:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234288.845794:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:2.0:1761234288.845799:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234288.845802:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234288.845803:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000c2788073. 00000100:00000001:2.0:1761234288.845808:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234288.845809:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845811:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234288.845814:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000007a0dd9b5 x1846784245235904/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234288.845821:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234288.845822:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234288.845823:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234288.845827:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234288.845828:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234288.845829:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234288.845832:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234288.845833:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000005851ba37. 02000000:00000001:2.0:1761234288.845836:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234288.845836:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234288.845838:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000007a0dd9b5. 00000100:00000001:2.0:1761234288.845841:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234288.845842:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845843:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234288.845885:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000086a5741d x1846784245235776/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234310 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234288.845892:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234288.845893:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234288.845895:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234288.845899:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234288.845901:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234288.845902:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234288.845904:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234288.845905:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078138607. 02000000:00000001:2.0:1761234288.845908:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234288.845910:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234288.845910:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000086a5741d. 00000100:00000001:2.0:1761234288.845914:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234288.845916:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845918:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234288.845930:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234288.845932:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234288.845934:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234288.845936:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234289.357038:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234289.357045:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234289.357046:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234289.357051:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234289.357056:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234289.357060:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234289.357062:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234289.357065:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234289.357067:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234289.740964:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234289.740969:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234289.740975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234289.740977:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234289.740980:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234289.740981:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234289.741050:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234289.741056:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234289.741060:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234290.380950:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234290.380959:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234290.380961:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234290.380968:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234290.380975:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234290.380979:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.380983:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234290.380986:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.380990:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234290.764990:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234290.764996:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234290.765003:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234290.765004:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234290.765006:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234290.765008:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234290.765065:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234290.765069:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234290.765073:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234290.893039:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234290.893055:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234290.893057:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234290.893061:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234290.893064:0:64357:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234290.893072:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234290.893082:0:64357:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6759 00000100:00080000:0.0:1761234290.893087:0:64357:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234290.893090:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893093:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234290.893096:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893098:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234290.893103:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234290.893105:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234290.893109:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234290.893112:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234290.893115:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234290.893118:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234290.893121:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893123:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234290.893125:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893127:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234290.893130:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234290.893134:0:64357:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893136:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234290.893140:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234290.893144:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234290.893147:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234290.893149:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234290.893152:0:64357:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234290.893164:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000e728cb50. 00000020:00000040:0.0:1761234290.893168:0:64357:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234290.893173:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234290.893174:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234290.893178:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234290.893181:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234290.893186:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234290.893195:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761234290.893202:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234290.893210:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234290.893220:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234290.893223:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234290.893226:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000e728cb50] to pc [ptlrpcd_rcv+-1] req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234290.893256:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234290.893258:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234290.893260:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234290.893264:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234290.893265:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234290.893267:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234290.893268:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234290.893271:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234290.893273:0:64357:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6759 00000100:00080000:0.0:1761234290.893275:0:64357:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234290.893278:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893279:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234290.893281:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893283:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234290.893286:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234290.893287:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234290.893288:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234290.893292:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234290.893294:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234290.893297:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234290.893300:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893302:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234290.893305:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893307:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234290.893309:0:64357:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234290.893312:0:64357:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893315:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234290.893316:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1761234290.893321:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1761234290.893324:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234290.893327:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.893328:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234290.893330:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234290.893333:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000008:00100000:0.0:1761234290.893335:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234290.893340:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234290.893341:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234290.893343:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234290.893348:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000dc434e0c. 00000020:00000040:0.0:1761234290.893351:0:64357:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234290.893356:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234290.893359:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.893360:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1761234290.893361:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234290.893363:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234290.893364:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234290.893367:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234290.893370:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:0.0:1761234290.893371:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 02000000:00000001:2.0:1761234290.893374:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234290.893377:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234290.893381:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000e728cb50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235968:192.168.201.124@tcp:250: 00000100:00100000:0.0:1761234290.893385:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.893391:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1761234290.893397:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1761234290.893397:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1761234290.893399:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234290.893401:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234290.893404:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000dc434e0c] to pc [ptlrpcd_rcv+-1] req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234290.893406:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234290.893413:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234290.893418:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234290.893425:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234290.893428:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000010:2.0:1761234290.893428:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00080000:0.0:1761234290.893430:0:64357:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234290.893435:0:64357:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234290.893436:0:64357:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761234290.893438:0:64357:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:2.0:1761234290.893438:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00080000:0.0:1761234290.893441:0:64357:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234290.893445:0:64357:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234290.893448:0:64357:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6759 00000100:00000200:2.0:1761234290.893448:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245235968, portal 25 00000100:00080000:0.0:1761234290.893452:0:64357:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1761234290.893453:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234290.893455:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246295744 : -104717463255872 : ffffa0c290cd04c0) 00000100:00000040:2.0:1761234290.893463:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234315 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:0.0:1761234290.893466:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893470:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234290.893474:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893479:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:2.0:1761234290.893480:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000020:00000001:0.0:1761234290.893481:0:64357:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234290.893483:0:64357:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234290.893485:0:64357:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234290.893487:0:64357:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000040:2.0:1761234290.893488:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000020:00000001:0.0:1761234290.893491:0:64357:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000400:00000010:2.0:1761234290.893493:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000020:00000001:0.0:1761234290.893494:0:64357:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000200:2.0:1761234290.893515:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245235968, offset 0 00000100:00000040:0.0:1761234290.893517:0:64357:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893521:0:64357:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234290.893523:0:64357:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893524:0:64357:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234290.893528:0:64357:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000200:2.0:1761234290.893530:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00080000:0.0:1761234290.893533:0:64357:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234290.893536:0:64357:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234290.893538:0:64357:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234290.893542:0:64357:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234290.893544:0:64357:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234290.893546:0:64357:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234290.893548:0:64357:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234290.893550:0:64357:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234290.893552:0:64357:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234290.893555:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00000010:0.0:1761234290.893556:0:64357:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000039bf264. 00000020:00000040:0.0:1761234290.893560:0:64357:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234290.893563:0:64357:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234290.893565:0:64357:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234290.893566:0:64357:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234290.893568:0:64357:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1761234290.893569:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 02000000:00000001:0.0:1761234290.893570:0:64357:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234290.893575:0:64357:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000400:00000200:2.0:1761234290.893576:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1761234290.893580:0:64357:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234290.893584:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00100000:0.0:1761234290.893588:0:64357:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234290.893594:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11904:11904:256:4294967295] 192.168.201.124@tcp LPNI seq info [11904:11904:8:4294967295] 00000100:00000001:0.0:1761234290.893603:0:64357:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234290.893606:0:64357:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234290.893609:0:64357:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000039bf264] to pc [ptlrpcd_rcv+-1] req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234290.893622:0:64357:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234290.893623:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:0.0:1761234290.893625:0:64357:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761234290.893631:0:64357:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6770) 00000800:00000200:2.0:1761234290.893636:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234290.893644:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234290.893656:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234290.893660:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1761234290.893668:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234290.893672:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 57 secs later) 00000800:00000100:2.0:1761234290.893677:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234290.893680:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234290.893687:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234290.893693:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234290.893700:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234290.893703:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234290.893710:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234290.893713:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234315 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.893723:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234290.893726:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234315 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.893733:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.893735:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234290.893741:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.893743:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234290.893747:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.893750:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234290.893753:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234290/real 1761234290] req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234290.893764:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234290.893766:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234290.893768:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234290.893770:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234290.893771:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234290.893773:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234290.893776:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234290.893785:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.893792:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234290.893793:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.893796:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234290.893797:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234290.893800:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.893806:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234290.893811:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234290.893818:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234290.893821:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.893823:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234290.893824:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.893827:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234290.893828:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234290.893831:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234290.893834:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234290.893837:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234290.893841:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234290.893886:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000e728cb50 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245235968:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234290.893893:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.893895:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234290.893898:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000e728cb50 x1846784245235968/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234290.893905:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234290.893908:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234290.893911:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234290.893915:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234290.893917:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234290.893918:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234290.893921:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234290.893923:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000005851ba37. 02000000:00000001:2.0:1761234290.893927:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234290.893929:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234290.893931:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000e728cb50. 00000100:00000001:2.0:1761234290.893934:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234290.893935:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.893937:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.893947:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234290.893948:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.893950:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234290.893952:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234290.893953:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234290.893956:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.893978:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234290.893979:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234290.893982:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234290.893983:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234290.893986:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000dc434e0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236032:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234290.893994:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234290.893996:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234290.893997:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234290.893999:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234290.894001:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234290.894003:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234290.894006:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234290.894010:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234290.894012:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236032, portal 10 00000100:00000001:2.0:1761234290.894014:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234290.894016:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246323712 : -104717463227904 : ffffa0c290cd7200) 00000100:00000040:2.0:1761234290.894023:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234315 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894030:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234290.894034:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234290.894036:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234290.894038:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245236032, offset 0 00000400:00000200:2.0:1761234290.894042:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234290.894048:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234290.894052:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234290.894055:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234290.894058:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234290.894061:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11905:11905:256:4294967295] 192.168.201.124@tcp LPNI seq info [11905:11905:8:4294967295] 00000400:00000200:2.0:1761234290.894073:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234290.894078:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234290.894081:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234290.894085:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234290.894087:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1761234290.894091:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234290.894094:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 57 secs later) 00000800:00000100:2.0:1761234290.894098:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234290.894099:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234290.894102:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234290.894106:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234290.894108:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234290.894110:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234290.894112:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234290.894115:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234315 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894123:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234290.894125:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234315 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894132:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.894134:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234290.894137:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.894138:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234290.894141:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.894142:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234290.894145:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234290/real 1761234290] req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234290.894153:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234290.894154:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234290.894156:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234290.894158:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234290.894160:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234290.894163:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234290.894166:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234290.894175:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894182:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234290.894183:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894185:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234290.894186:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234290.894189:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894196:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234290.894201:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234290.894208:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234290.894210:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894212:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234290.894213:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894215:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234290.894216:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234290.894219:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234290.894222:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234290.894224:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234290.894228:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234290.894236:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000dc434e0c pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236032:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234290.894240:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234290.894242:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894249:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234290.894250:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234290.894252:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234290.894254:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234290.894256:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000039bf264 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236096:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234290.894259:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234290.894261:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234290.894262:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234290.894264:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234290.894267:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234290.894269:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234290.894272:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234290.894276:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234290.894279:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236096, portal 4 00000100:00000001:2.0:1761234290.894280:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234290.894281:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246311552 : -104717463240064 : ffffa0c290cd4280) 00000100:00000040:2.0:1761234290.894286:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234315 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894292:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234290.894295:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234290.894297:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234290.894299:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245236096, offset 0 00000400:00000200:2.0:1761234290.894303:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234290.894309:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234290.894312:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234290.894314:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234290.894317:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234290.894319:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11906:11906:256:4294967295] 192.168.201.124@tcp LPNI seq info [11906:11906:8:4294967295] 00000400:00000200:2.0:1761234290.894327:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234290.894331:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234290.894334:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234290.894339:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234290.894341:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 57 secs later) 00000800:00000200:2.1:1761234290.894345:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234290.894348:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 57 secs later) 00000800:00000100:2.0:1761234290.894352:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234290.894354:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234290.894357:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234290.894361:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234290.894363:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234290.894365:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234290.894367:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234290.894370:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234315 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894378:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234290.894380:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234315 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894393:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.894395:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234290.894398:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.894399:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234290.894401:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.894403:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234290.894406:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234290/real 1761234290] req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234290.894413:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234290.894414:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234290.894416:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234290.894418:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234290.894419:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234290.894421:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234290.894423:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234290.894430:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894436:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234290.894437:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894439:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234290.894440:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234290.894443:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234290.894450:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234290.894453:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234290.894460:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234290.894463:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894464:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234290.894465:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894467:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234290.894468:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234290.894470:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234290.894473:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234290.894475:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234290.894478:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234290.894485:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000039bf264 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236096:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234290.894492:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894494:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234290.894507:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000dc434e0c x1846784245236032/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234290.894514:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234290.894516:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234290.894517:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234290.894521:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234290.894522:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234290.894523:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234290.894525:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234290.894526:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:2.0:1761234290.894529:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234290.894530:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234290.894532:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000dc434e0c. 00000100:00000001:2.0:1761234290.894534:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234290.894536:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894537:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234290.894540:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000039bf264 x1846784245236096/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234315 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234290.894546:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234290.894547:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234290.894548:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000078016eab. 02000000:00000001:2.0:1761234290.894550:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234290.894551:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234290.894552:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234290.894554:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234290.894555:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000007c64e188. 02000000:00000001:2.0:1761234290.894558:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234290.894559:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234290.894560:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000039bf264. 00000100:00000001:2.0:1761234290.894562:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234290.894563:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894565:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234290.894569:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234290.894570:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234290.894572:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234290.894574:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234291.404964:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234291.404972:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234291.404974:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234291.404979:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234291.404986:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234291.404990:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234291.404994:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234291.404996:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234291.404999:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788942:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234291.788950:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234291.788952:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234291.788955:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788960:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234291.788962:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788964:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788966:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788969:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234291.788971:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234291.788972:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234291.788977:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234291.788978:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234291.788979:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788980:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234291.788981:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788982:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788983:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788986:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234291.788987:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234291.788988:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234291.788990:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234291.788990:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234291.788991:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788992:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234291.788992:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788993:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234291.788996:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234291.789042:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234291.789045:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234291.789048:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234292.429013:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234292.429019:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234292.429021:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234292.429026:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234292.429032:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234292.429035:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234292.429037:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234292.429040:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234292.429042:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234292.813036:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234292.813044:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234292.813053:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234292.813054:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234292.813058:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234292.813059:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234292.813141:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234292.813147:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234292.813152:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234293.453020:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234293.453026:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234293.453028:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234293.453033:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234293.453046:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234293.453052:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234293.453055:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234293.453058:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234293.453061:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234293.836946:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234293.836952:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234293.836958:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234293.836959:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234293.836962:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234293.836963:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234293.837013:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234293.837016:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234293.837020:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234294.476977:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234294.476983:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234294.476985:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234294.476991:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234294.476996:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234294.476999:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234294.477001:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234294.477005:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234294.477009:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234294.860991:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234294.860996:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234294.861003:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234294.861004:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234294.861007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234294.861008:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234294.861068:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234294.861073:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234294.861076:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234295.500953:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234295.500959:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234295.500961:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234295.500965:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234295.500970:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234295.500973:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234295.500975:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234295.500978:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234295.500980:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234295.885021:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234295.885026:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234295.885033:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234295.885034:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234295.885037:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234295.885037:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234295.885134:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234295.885137:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234295.885142:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1761234296.012956:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234296.012965:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234296.012966:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:3.0:1761234296.012968:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234296.012970:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234296.012974:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234296.012979:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6765 00000100:00080000:3.0:1761234296.012982:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234296.012984:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.012987:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234296.012988:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.012990:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234296.012993:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234296.012994:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:3.0:1761234296.012996:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234296.012999:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:3.0:1761234296.013000:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:3.0:1761234296.013002:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:3.0:1761234296.013003:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013004:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234296.013005:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013006:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234296.013008:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:3.0:1761234296.013011:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013012:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234296.013014:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234296.013017:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234296.013018:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013020:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:3.0:1761234296.013021:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234296.013029:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000009ecb9068. 00000020:00000040:3.0:1761234296.013032:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:3.0:1761234296.013035:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234296.013036:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013038:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234296.013040:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234296.013043:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234296.013050:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000001c1b0e94. 00000100:00000001:3.0:1761234296.013071:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234296.013076:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234296.013087:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234296.013089:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234296.013092:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000009ecb9068] to pc [ptlrpcd_rcv+-1] req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234296.013112:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013114:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234296.013115:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1761234296.013118:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234296.013134:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1761234296.013135:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234296.013136:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234296.013137:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:3.0:1761234296.013139:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6765 00000100:00080000:3.0:1761234296.013140:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1761234296.013141:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013142:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234296.013143:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013144:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:3.0:1761234296.013145:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234296.013145:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:3.0:1761234296.013146:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:3.0:1761234296.013147:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:3.0:1761234296.013148:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:3.0:1761234296.013149:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:3.0:1761234296.013151:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013152:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234296.013153:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013153:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:3.0:1761234296.013155:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:3.0:1761234296.013156:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013158:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234296.013159:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1761234296.013163:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234296.013164:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013165:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234296.013167:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1761234296.013169:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013171:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1761234296.013173:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000194cd3a7. 00000020:00000040:3.0:1761234296.013174:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:3.0:1761234296.013175:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234296.013176:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013177:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234296.013178:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1761234296.013179:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234296.013181:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074a27ae0. 00000100:00000001:3.0:1761234296.013183:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1761234296.013185:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234296.013198:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234296.013199:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234296.013202:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000194cd3a7] to pc [ptlrpcd_rcv+-1] req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013206:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1761234296.013207:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013210:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:3.0:1761234296.013211:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1761234296.013212:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234296.013214:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1761234296.013217:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:3.0:1761234296.013219:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000040:2.0:1761234296.013221:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:3.0:1761234296.013221:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1761234296.013223:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1761234296.013227:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00000001:2.0:1761234296.013231:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1761234296.013231:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6765 00000100:00000001:2.0:1761234296.013233:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1761234296.013234:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1761234296.013237:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1761234296.013238:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000001:2.0:1761234296.013241:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013241:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234296.013244:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013246:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00100000:2.0:1761234296.013247:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245236160:192.168.201.124@tcp:250: 00000020:00000001:3.0:1761234296.013249:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:3.0:1761234296.013250:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:3.0:1761234296.013252:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000100:00000001:2.0:1761234296.013253:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1761234296.013255:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:3.0:1761234296.013257:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:2.0:1761234296.013258:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1761234296.013259:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 02000000:00000001:2.0:1761234296.013260:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234296.013263:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000001:2.0:1761234296.013264:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1761234296.013266:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1761234296.013268:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013269:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 02000000:00000010:2.0:1761234296.013270:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 00000020:00000040:3.0:1761234296.013271:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 02000000:00000001:2.0:1761234296.013273:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1761234296.013275:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013278:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1761234296.013279:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1761234296.013280:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 02000000:00000001:3.0:1761234296.013281:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1761234296.013284:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013285:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:3.0:1761234296.013286:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000400:00000010:2.0:1761234296.013287:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000008:00000001:3.0:1761234296.013288:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013290:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234296.013293:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236160, portal 25 00000100:00000010:3.0:1761234296.013294:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000a52d2576. 00000100:00000001:2.0:1761234296.013298:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000020:00000040:3.0:1761234296.013299:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000001:2.0:1761234296.013301:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091118656 : -104717618432960 : ffffa0c2878d3440) 02000000:00000001:3.0:1761234296.013302:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:3.0:1761234296.013303:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013306:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1761234296.013307:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:2.0:1761234296.013308:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234321 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 02000000:00000001:3.0:1761234296.013309:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1761234296.013314:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000074003a94. 00000100:00000001:3.0:1761234296.013317:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013318:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234296.013324:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00100000:3.0:1761234296.013325:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000010:2.0:1761234296.013327:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234296.013332:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245236160, offset 0 00000100:00000001:3.0:1761234296.013333:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1761234296.013335:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1761234296.013337:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000a52d2576] to pc [ptlrpcd_rcv+-1] req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234296.013343:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:3.0:1761234296.013349:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1761234296.013350:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:3.0:1761234296.013353:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6775) 00000400:00000200:2.0:1761234296.013359:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234296.013365:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234296.013369:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234296.013372:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234296.013375:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11907:11907:256:4294967295] 192.168.201.124@tcp LPNI seq info [11907:11907:8:4294967295] 00000400:00000200:2.0:1761234296.013382:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234296.013386:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234296.013389:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234296.013394:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234296.013398:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761234296.013418:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234296.013421:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1761234296.013423:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234296.013425:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234296.013430:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234296.013433:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234296.013437:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234296.013438:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234296.013442:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234296.013444:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234321 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013451:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234296.013453:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234321 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013457:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013459:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234296.013462:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013463:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234296.013465:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013467:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234296.013469:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234296/real 1761234296] req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234296.013474:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234296.013475:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234296.013476:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234296.013478:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234296.013479:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234296.013480:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234296.013481:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234296.013486:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013489:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234296.013490:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013492:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234296.013493:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234296.013495:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013498:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234296.013501:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234296.013505:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234296.013507:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013508:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234296.013508:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013510:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234296.013511:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234296.013517:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234296.013519:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234296.013520:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234296.013523:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234296.013528:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000009ecb9068 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245236160:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234296.013530:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234296.013532:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013536:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234296.013536:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234296.013538:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234296.013539:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234296.013540:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236224:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234296.013542:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234296.013543:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234296.013544:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234296.013545:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234296.013547:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234296.013548:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234296.013549:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234296.013551:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234296.013553:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236224, portal 10 00000100:00000001:2.0:1761234296.013554:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234296.013554:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091134464 : -104717618417152 : ffffa0c2878d7200) 00000100:00000040:2.0:1761234296.013558:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234321 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013562:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234296.013563:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234296.013565:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234296.013566:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245236224, offset 0 00000400:00000200:2.0:1761234296.013569:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234296.013571:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234296.013573:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234296.013575:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234296.013576:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234296.013578:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11908:11908:256:4294967295] 192.168.201.124@tcp LPNI seq info [11908:11908:8:4294967295] 00000400:00000200:2.0:1761234296.013583:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234296.013585:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234296.013587:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234296.013589:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234296.013591:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761234296.013593:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234296.013595:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1761234296.013597:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234296.013598:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234296.013599:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234296.013601:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234296.013603:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234296.013603:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234296.013605:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234296.013606:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234321 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013610:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234296.013611:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234321 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013615:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013616:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234296.013617:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013618:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234296.013619:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013620:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234296.013622:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234296/real 1761234296] req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234296.013626:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234296.013627:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234296.013628:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234296.013629:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234296.013629:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234296.013630:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234296.013631:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234296.013635:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013638:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234296.013639:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013640:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234296.013640:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234296.013642:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013645:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234296.013647:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234296.013650:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234296.013651:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013652:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234296.013652:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013653:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234296.013654:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234296.013655:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234296.013657:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234296.013658:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234296.013660:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234296.013664:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000194cd3a7 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236224:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234296.013666:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013667:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234296.013668:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000009ecb9068 x1846784245236160/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234296.013672:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234296.013673:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234296.013675:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234296.013677:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234296.013678:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234296.013679:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234296.013680:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234296.013682:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000001c1b0e94. 02000000:00000001:2.0:1761234296.013685:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234296.013686:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234296.013687:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000009ecb9068. 00000100:00000001:2.0:1761234296.013689:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234296.013690:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013691:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234296.013692:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000194cd3a7 x1846784245236224/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234296.013696:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234296.013697:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234296.013697:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234296.013699:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234296.013699:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234296.013700:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234296.013701:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234296.013701:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074a27ae0. 02000000:00000001:2.0:1761234296.013703:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234296.013704:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234296.013704:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000194cd3a7. 00000100:00000001:2.0:1761234296.013706:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234296.013706:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013707:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013713:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234296.013715:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013716:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234296.013716:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234296.013717:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234296.013718:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013722:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234296.013723:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234296.013724:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234296.013725:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234296.013726:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236288:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234296.013728:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234296.013729:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234296.013730:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234296.013731:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234296.013732:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234296.013733:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234296.013735:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234296.013736:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234296.013738:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236288, portal 4 00000100:00000001:2.0:1761234296.013739:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234296.013739:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356091112576 : -104717618439040 : ffffa0c2878d1c80) 00000100:00000040:2.0:1761234296.013742:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234321 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013746:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234296.013747:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234296.013748:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234296.013749:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245236288, offset 0 00000400:00000200:2.0:1761234296.013752:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234296.013755:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234296.013757:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234296.013758:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234296.013760:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234296.013761:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11909:11909:256:4294967295] 192.168.201.124@tcp LPNI seq info [11909:11909:8:4294967295] 00000400:00000200:2.0:1761234296.013771:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234296.013773:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234296.013775:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234296.013777:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234296.013779:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000200:2.1:1761234296.013781:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234296.013782:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 52 secs later) 00000800:00000100:2.0:1761234296.013784:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234296.013786:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234296.013787:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234296.013789:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234296.013790:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234296.013791:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234296.013792:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234296.013793:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234321 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013798:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234296.013799:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234321 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013802:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013803:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234296.013805:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013806:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234296.013807:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013808:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234296.013809:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234296/real 1761234296] req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234296.013818:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234296.013818:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234296.013819:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234296.013820:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234296.013821:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234296.013822:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234296.013823:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234296.013827:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013830:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234296.013830:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013831:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234296.013832:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234296.013833:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234296.013837:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234296.013838:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234296.013842:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234296.013864:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013866:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234296.013866:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013868:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234296.013868:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234296.013870:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234296.013872:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234296.013873:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234296.013875:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234296.013884:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000a52d2576 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236288:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234296.013886:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013887:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234296.013888:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000a52d2576 x1846784245236288/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234321 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234296.013892:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234296.013892:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234296.013893:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000cf0983ca. 02000000:00000001:2.0:1761234296.013895:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234296.013895:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234296.013896:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234296.013897:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234296.013897:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000074003a94. 02000000:00000001:2.0:1761234296.013898:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234296.013899:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234296.013900:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000a52d2576. 00000100:00000001:2.0:1761234296.013902:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234296.013903:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013904:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.013906:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234296.013907:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.013908:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234296.013909:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234296.524916:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234296.524922:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234296.524923:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234296.524927:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234296.524931:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234296.524934:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.524935:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234296.524937:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234296.524939:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234296.908974:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234296.908979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234296.908987:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234296.908989:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234296.908992:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234296.908994:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234296.909054:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234296.909057:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234296.909060:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234297.548939:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234297.548946:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234297.548949:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234297.548954:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234297.548959:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234297.548962:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234297.548965:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234297.548967:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234297.548969:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234297.932996:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234297.933002:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234297.933010:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234297.933011:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234297.933015:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234297.933016:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234297.933096:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234297.933099:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234297.933103:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234298.572949:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234298.572956:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234298.572958:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234298.572963:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234298.572969:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234298.572972:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234298.572975:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234298.572977:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234298.572979:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234298.956997:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234298.957002:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234298.957007:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234298.957009:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234298.957012:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234298.957014:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234298.957071:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234298.957074:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234298.957077:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234299.596931:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234299.596935:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234299.596936:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234299.596940:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234299.596945:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234299.596947:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234299.596949:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234299.596950:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234299.596952:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234299.980975:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234299.980979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234299.980983:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234299.980984:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234299.980986:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234299.980987:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234299.981052:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234299.981055:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234299.981058:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234300.620947:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234300.620953:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234300.620954:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234300.620958:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234300.620964:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234300.620967:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234300.620970:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234300.620972:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234300.620974:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.004962:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234301.004968:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234301.004970:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234301.004972:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.004977:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234301.004978:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.004980:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.004981:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.004984:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234301.004985:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234301.004986:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234301.004990:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234301.004992:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234301.004992:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.004993:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234301.004994:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.004995:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.004996:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.004999:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234301.005001:0:92377:0:(ldlm_request.c:2073:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1761234301.005001:0:92377:0:(ldlm_request.c:1878:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1761234301.005003:0:92377:0:(ldlm_lock.c:208:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1761234301.005004:0:92377:0:(ldlm_lock.c:248:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1761234301.005005:0:92377:0:(ldlm_request.c:2041:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.005006:0:92377:0:(ldlm_lockd.c:2211:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1761234301.005006:0:92377:0:(ldlm_lockd.c:2214:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.005007:0:92377:0:(ldlm_request.c:2083:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234301.005008:0:92377:0:(ldlm_pool.c:517:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234301.005095:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234301.005098:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234301.005101:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234301.133054:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234301.133067:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234301.133069:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234301.133072:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234301.133075:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234301.133081:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234301.133089:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6770 00000100:00080000:0.0:1761234301.133093:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234301.133095:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133098:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234301.133100:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133102:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234301.133106:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234301.133108:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234301.133111:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234301.133114:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234301.133116:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234301.133117:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234301.133119:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133120:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234301.133122:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133123:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234301.133125:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234301.133128:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133130:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234301.133133:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234301.133136:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234301.133138:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234301.133140:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234301.133141:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234301.133153:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000efaa8dcd. 00000020:00000040:0.0:1761234301.133156:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234301.133160:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234301.133161:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234301.133165:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234301.133168:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234301.133172:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234301.133183:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000040aa0f54. 00000100:00000001:0.0:1761234301.133189:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234301.133196:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234301.133208:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234301.133210:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234301.133213:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000efaa8dcd] to pc [ptlrpcd_rcv+-1] req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234301.133237:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234301.133238:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234301.133240:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234301.133243:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234301.133243:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234301.133244:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234301.133245:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234301.133247:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234301.133249:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6770 00000100:00080000:0.0:1761234301.133250:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234301.133252:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133253:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234301.133253:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133254:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234301.133255:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234301.133256:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000020:00000001:0.0:1761234301.133257:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234301.133258:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234301.133259:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234301.133260:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000040:0.0:1761234301.133262:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133263:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234301.133264:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133265:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234301.133267:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 00000100:00080000:0.0:1761234301.133269:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133271:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234301.133272:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234301.133279:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234301.133281:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234301.133282:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:2.0:1761234301.133284:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1761234301.133286:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234301.133290:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133290:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1761234301.133294:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133294:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000010:0.0:1761234301.133298:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000000080840a. 00000020:00000040:0.0:1761234301.133301:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000100:00000040:2.0:1761234301.133315:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133324:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1761234301.133326:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:2.0:1761234301.133327:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234301.133329:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234301.133331:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234301.133332:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1761234301.133333:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1761234301.133334:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234301.133336:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234301.133338:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000007c64e188. 00000100:00100000:2.0:1761234301.133342:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000efaa8dcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245236352:192.168.201.124@tcp:250: 00000100:00000001:0.0:1761234301.133343:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133349:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1761234301.133351:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000001:2.0:1761234301.133353:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234301.133356:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234301.133358:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1761234301.133360:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1761234301.133361:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234301.133364:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000000080840a] to pc [ptlrpcd_rcv+-1] req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 02000000:00000010:2.0:1761234301.133367:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 00000100:00000001:0.0:1761234301.133369:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234301.133370:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234301.133374:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234301.133376:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234301.133378:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000400:00000010:2.0:1761234301.133379:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000100:00080000:0.0:1761234301.133380:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1761234301.133382:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234301.133384:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234301.133386:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234301.133389:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6770 00000400:00000010:2.0:1761234301.133389:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00080000:0.0:1761234301.133393:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234301.133395:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133397:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234301.133398:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000200:2.0:1761234301.133399:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236352, portal 25 00000100:00000001:0.0:1761234301.133400:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234301.133404:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000100:00000001:2.0:1761234301.133405:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000020:00000040:0.0:1761234301.133406:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000100:00000001:2.0:1761234301.133407:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246318848 : -104717463232768 : ffffa0c290cd5f00) 00000020:00000001:0.0:1761234301.133408:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234301.133410:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000020:00000001:0.0:1761234301.133413:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761234301.133416:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000040:2.0:1761234301.133416:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234326 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000040:0.0:1761234301.133422:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133425:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234301.133428:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234301.133430:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000001:0.0:1761234301.133431:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234301.133433:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000100:00000040:2.0:1761234301.133438:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00080000:0.0:1761234301.133439:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133443:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234301.133443:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 02000000:00000001:0.0:1761234301.133445:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234301.133448:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1761234301.133448:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245236352, offset 0 02000000:00000001:0.0:1761234301.133451:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234301.133453:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234301.133454:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234301.133458:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234301.133459:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133460:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234301.133465:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000581658b4. 00000020:00000040:0.0:1761234301.133466:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 02000000:00000001:0.0:1761234301.133468:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234301.133468:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234301.133469:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234301.133470:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234301.133472:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234301.133474:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00000001:0.0:1761234301.133476:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234301.133479:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000100:00100000:0.0:1761234301.133482:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234301.133488:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234301.133489:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234301.133491:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000040:0.0:1761234301.133494:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000581658b4] to pc [ptlrpcd_rcv+-1] req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234301.133498:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1761234301.133503:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234301.133503:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000001:0.0:1761234301.133507:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000400:00000200:2.0:1761234301.133509:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11910:11910:256:4294967295] 192.168.201.124@tcp LPNI seq info [11910:11910:8:4294967295] 00000100:00000040:0.0:1761234301.133511:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6780) 00000400:00000200:2.0:1761234301.133523:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234301.133529:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234301.133535:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234301.133544:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234301.133547:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1761234301.133553:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234301.133555:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1761234301.133559:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234301.133561:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234301.133568:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234301.133573:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234301.133576:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234301.133578:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234301.133584:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234301.133586:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234326 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133593:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234301.133595:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234326 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133600:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133601:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234301.133606:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133608:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234301.133610:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133612:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234301.133614:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234301/real 1761234301] req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234301.133621:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234301.133623:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234301.133624:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234301.133625:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234301.133626:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234301.133627:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234301.133629:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234301.133634:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133638:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234301.133640:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.133641:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234301.133643:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234301.133645:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133648:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234301.133651:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234301.133655:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234301.133657:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.133658:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234301.133658:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.133660:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234301.133661:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234301.133663:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234301.133666:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234301.133668:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234301.133670:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234301.133680:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000efaa8dcd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245236352:192.168.201.124@tcp:250: 00000100:00000001:2.0:1761234301.133683:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.133685:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234301.133686:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000efaa8dcd x1846784245236352/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234301.133690:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234301.133693:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234301.133695:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234301.133697:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234301.133698:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234301.133699:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234301.133700:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234301.133702:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000040aa0f54. 02000000:00000001:2.0:1761234301.133706:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234301.133707:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234301.133708:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000efaa8dcd. 00000100:00000001:2.0:1761234301.133711:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234301.133712:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.133713:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.133722:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234301.133723:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133725:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234301.133726:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234301.133727:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234301.133729:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133734:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234301.133734:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234301.133736:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234301.133737:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234301.133738:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000000080840a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236416:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234301.133740:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234301.133742:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234301.133742:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234301.133744:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234301.133745:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234301.133747:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234301.133749:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234301.133751:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234301.133752:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236416, portal 10 00000100:00000001:2.0:1761234301.133753:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234301.133754:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246309120 : -104717463242496 : ffffa0c290cd3900) 00000100:00000040:2.0:1761234301.133759:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234326 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133763:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234301.133765:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234301.133766:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234301.133767:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245236416, offset 0 00000400:00000200:2.0:1761234301.133769:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234301.133772:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234301.133775:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234301.133776:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234301.133778:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234301.133779:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11911:11911:256:4294967295] 192.168.201.124@tcp LPNI seq info [11911:11911:8:4294967295] 00000400:00000200:2.0:1761234301.133785:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234301.133788:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234301.133789:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234301.133791:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234301.133793:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1761234301.133795:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234301.133797:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1761234301.133799:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234301.133801:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234301.133803:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234301.133804:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234301.133807:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234301.133808:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234301.133809:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234301.133810:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234326 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133815:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234301.133816:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234326 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133819:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133820:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234301.133822:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133823:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234301.133824:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133825:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234301.133827:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234301/real 1761234301] req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234301.133832:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234301.133833:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234301.133833:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234301.133834:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234301.133835:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234301.133836:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234301.133837:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234301.133841:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133868:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234301.133869:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.133870:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234301.133871:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234301.133873:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133878:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234301.133880:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234301.133884:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234301.133886:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.133887:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234301.133887:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.133888:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234301.133889:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234301.133890:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234301.133892:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234301.133893:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234301.133895:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234301.133899:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000000080840a pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236416:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234301.133902:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234301.133903:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133906:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234301.133907:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234301.133908:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234301.133909:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234301.133910:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000581658b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236480:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234301.133912:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234301.133914:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234301.133915:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234301.133916:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234301.133918:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000003562879b. 02000000:00000001:2.0:1761234301.133919:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234301.133921:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234301.133923:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234301.133925:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236480, portal 4 00000100:00000001:2.0:1761234301.133927:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234301.133927:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246320064 : -104717463231552 : ffffa0c290cd63c0) 00000100:00000040:2.0:1761234301.133930:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234326 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133935:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234301.133936:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234301.133937:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234301.133939:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245236480, offset 0 00000400:00000200:2.0:1761234301.133941:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234301.133944:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234301.133946:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234301.133947:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234301.133949:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234301.133951:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11912:11912:256:4294967295] 192.168.201.124@tcp LPNI seq info [11912:11912:8:4294967295] 00000400:00000200:2.0:1761234301.133956:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234301.133959:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234301.133960:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234301.133962:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234301.133963:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000200:2.1:1761234301.133966:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234301.133967:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 47 secs later) 00000800:00000100:2.0:1761234301.133970:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234301.133971:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234301.133973:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234301.133975:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234301.133977:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234301.133978:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234301.133979:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234301.133980:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234326 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133985:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234301.133987:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234326 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.133991:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133992:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234301.133993:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133994:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234301.133995:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.133996:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234301.133998:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234301/real 1761234301] req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234301.134002:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234301.134003:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234301.134004:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234301.134005:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234301.134006:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234301.134006:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234301.134008:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234301.134011:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.134014:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234301.134015:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.134016:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234301.134017:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234301.134018:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234301.134021:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234301.134023:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234301.134026:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234301.134027:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.134028:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234301.134029:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.134030:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234301.134031:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234301.134033:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234301.134034:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234301.134035:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234301.134037:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234301.134040:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000581658b4 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236480:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234301.134042:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.134043:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234301.134044:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000000080840a x1846784245236416/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234301.134048:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234301.134048:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234301.134049:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234301.134050:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234301.134051:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234301.134051:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234301.134053:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234301.134053:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000007c64e188. 02000000:00000001:2.0:1761234301.134055:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234301.134056:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234301.134056:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000000080840a. 00000100:00000001:2.0:1761234301.134058:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234301.134059:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.134060:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234301.134061:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000581658b4 x1846784245236480/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234326 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234301.134065:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234301.134065:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234301.134066:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000003562879b. 02000000:00000001:2.0:1761234301.134068:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234301.134069:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234301.134070:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234301.134071:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234301.134072:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:2.0:1761234301.134073:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234301.134074:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234301.134074:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000581658b4. 00000100:00000001:2.0:1761234301.134075:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234301.134076:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.134077:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.134079:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234301.134079:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.134080:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234301.134081:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234301.644981:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234301.644990:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234301.644992:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234301.644997:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234301.645003:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234301.645007:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.645011:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234301.645014:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234301.645017:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234302.029012:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234302.029018:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234302.029024:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234302.029025:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234302.029028:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234302.029029:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234302.029091:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234302.029095:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234302.029100:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234302.668995:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234302.669004:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234302.669005:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234302.669012:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234302.669018:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234302.669023:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234302.669026:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234302.669029:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234302.669033:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234303.052973:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234303.052979:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234303.052986:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234303.052986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234303.052989:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234303.052990:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234303.053077:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234303.053081:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234303.053084:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234303.692999:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234303.693005:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234303.693007:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234303.693013:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234303.693018:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234303.693022:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234303.693024:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234303.693027:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234303.693029:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234304.077009:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234304.077014:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234304.077021:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234304.077022:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234304.077024:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234304.077025:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234304.077124:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234304.077128:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234304.077132:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234304.716954:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234304.716960:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234304.716961:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234304.716965:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234304.716971:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234304.716974:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234304.716976:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234304.716978:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234304.716980:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234305.100970:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234305.100975:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234305.100982:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234305.100983:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234305.100985:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234305.100986:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234305.101082:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234305.101087:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234305.101093:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234305.741007:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234305.741016:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234305.741017:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234305.741024:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234305.741030:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234305.741035:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234305.741039:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234305.741042:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234305.741045:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234306.125009:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234306.125015:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234306.125022:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234306.125023:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234306.125027:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234306.125028:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234306.125159:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234306.125164:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234306.125170:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1761234306.253000:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) f94b4173-c622-408c-b948-b77c5482236b->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234306.253013:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234306.253015:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) MGS: starting recovery 00000100:00000001:0.0:1761234306.253018:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234306.253020:0:468:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234306.253026:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234306.253034:0:468:0:(import.c:534:import_select_connection()) MGC192.168.201.124@tcp: connect to NID 192.168.201.124@tcp last attempt 6775 00000100:00080000:0.0:1761234306.253038:0:468:0:(import.c:581:import_select_connection()) MGC192.168.201.124@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234306.253041:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234306.253044:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234306.253046:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234306.253049:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234306.253054:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234306.253056:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9c87 00000020:00000001:0.0:1761234306.253059:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234306.253062:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000007389d189 refcount=3 00000020:00000001:0.0:1761234306.253065:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000020:00000001:0.0:1761234306.253067:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356180176896 : -104717529374720 : ffffa0c28cdc2000) 00000100:00000040:0.0:1761234306.253071:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234306.253074:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1761234306.253076:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:0.0:1761234306.253078:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234306.253081:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000007389d189 : new refcount 2 00000100:00080000:0.0:1761234306.253085:0:468:0:(import.c:615:import_select_connection()) MGC192.168.201.124@tcp: import 0000000014ae8b98 using connection MGC192.168.201.124@tcp_0/192.168.201.124@tcp 00000100:00000001:0.0:1761234306.253088:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234306.253091:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1761234306.253094:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1761234306.253097:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234306.253100:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000100:00000001:0.0:1761234306.253102:0:468:0:(obd_class.h:861:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1761234306.253112:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 000000003c218247. 00000020:00000040:0.0:1761234306.253116:0:468:0:(genops.c:1191:class_import_get()) import 0000000014ae8b98 refcount=4 obd=MGC192.168.201.124@tcp 02000000:00000001:0.0:1761234306.253120:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234306.253121:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234306.253125:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1761234306.253128:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1761234306.253131:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1761234306.253140:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 000000005851ba37. 00000100:00000001:0.0:1761234306.253148:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234306.253156:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234306.253168:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234306.253170:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234306.253175:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [000000003c218247] to pc [ptlrpcd_rcv+-1] req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234306.253201:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234306.253204:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00080000:0.0:1761234306.253223:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234306.253228:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000100:00080000:0.0:1761234306.253229:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1761234306.253231:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234306.253234:0:468:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1761234306.253238:0:468:0:(import.c:519:import_select_connection()) Process entered 00000100:00080000:0.0:1761234306.253242:0:468:0:(import.c:534:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6775 00000100:00080000:0.0:1761234306.253245:0:468:0:(import.c:581:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1761234306.253248:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234306.253250:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1761234306.253251:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1761234306.253261:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1761234306.253262:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234306.253266:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1761234306.253267:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000001:0.0:1761234306.253273:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234306.253275:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9ccd 00000100:00000040:2.0:1761234306.253277:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000020:00000001:0.0:1761234306.253278:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234306.253282:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 000000000f770f7e refcount=5 00000020:00000001:0.0:1761234306.253288:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000020:00000001:0.0:1761234306.253294:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356087261184 : -104717622290432 : ffffa0c287525800) 00000100:00000001:2.0:1761234306.253303:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234306.253305:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234306.253308:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 02000000:00000001:2.0:1761234306.253311:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1761234306.253312:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1761234306.253316:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1761234306.253319:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00100000:2.0:1761234306.253324:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@000000003c218247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245236544:192.168.201.124@tcp:250: 00000100:00000001:0.0:1761234306.253326:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000001:2.0:1761234306.253331:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1761234306.253336:0:468:0:(genops.c:979:class_export_put()) PUTting export 000000000f770f7e : new refcount 4 02000000:00000001:2.0:1761234306.253338:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234306.253342:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234306.253346:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1761234306.253347:0:468:0:(import.c:615:import_select_connection()) lustre-MDT0000-mdc-ffffa0c283fe8000: import 00000000ef40b234 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000100:00000001:0.0:1761234306.253352:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234306.253356:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:2.0:1761234306.253356:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234306.253362:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1761234306.253366:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1761234306.253368:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 02000000:00000001:0.0:1761234306.253373:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1761234306.253376:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000400:00000010:2.0:1761234306.253377:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000008:00100000:0.0:1761234306.253380:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1761234306.253386:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234306.253387:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236544, portal 25 00000100:00000001:0.0:1761234306.253389:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.253391:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234306.253395:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246306688 : -104717463244928 : ffffa0c290cd2f80) 00000100:00000010:0.0:1761234306.253398:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 0000000070caacfc. 00000100:00000040:2.0:1761234306.253401:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234331 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000020:00000040:0.0:1761234306.253404:0:468:0:(genops.c:1191:class_import_get()) import 00000000ef40b234 refcount=205 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 02000000:00000001:0.0:1761234306.253414:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 00000100:00000001:2.0:1761234306.253417:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 02000000:00000001:0.0:1761234306.253419:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234306.253426:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234306.253427:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1761234306.253431:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 02000000:00000001:0.0:1761234306.253432:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1761234306.253435:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1846784245236544, offset 0 02000000:00000001:0.0:1761234306.253436:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234306.253444:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 02000000:00000010:0.0:1761234306.253445:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 0000000078138607. 00000100:00000001:0.0:1761234306.253454:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1761234306.253463:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234306.253465:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234306.253476:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000100:00000001:0.0:1761234306.253478:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1761234306.253482:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1761234306.253483:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234306.253489:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000100:00000040:0.0:1761234306.253497:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [0000000070caacfc] to pc [ptlrpcd_rcv+-1] req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000400:00000200:2.0:1761234306.253498:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11913:11913:256:4294967295] 192.168.201.124@tcp LPNI seq info [11913:11913:8:4294967295] 00000400:00000200:2.0:1761234306.253522:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000100:00000001:0.0:1761234306.253525:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1761234306.253529:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000100:00000001:0.0:1761234306.253531:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000800:00000010:2.0:1761234306.253536:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000100:00080000:0.0:1761234306.253537:0:468:0:(pinger.c:251:ptlrpc_pinger_process_import()) e8fc2de5-c58c-4d00-8403-987fe90bf686->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1761234306.253545:0:468:0:(recover.c:55:ptlrpc_initiate_recovery()) Process entered 00000800:00000200:2.0:1761234306.253547:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00080000:0.0:1761234306.253548:0:468:0:(recover.c:57:ptlrpc_initiate_recovery()) lustre-OST0000_UUID: starting recovery 00000800:00000200:2.0:1761234306.253552:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000100:00000001:0.0:1761234306.253553:0:468:0:(import.c:686:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1761234306.253555:0:468:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:2.1:1761234306.253565:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000100:00000001:0.0:1761234306.253567:0:468:0:(import.c:519:import_select_connection()) Process entered 00000800:00000200:2.1:1761234306.253568:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000100:00080000:0.0:1761234306.253579:0:468:0:(import.c:534:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: connect to NID 192.168.201.124@tcp last attempt 6775 00000800:00000100:2.0:1761234306.253579:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234306.253583:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000100:00080000:0.0:1761234306.253586:0:468:0:(import.c:581:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1761234306.253589:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1761234306.253595:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000400:00000200:2.0:1761234306.253598:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1761234306.253604:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1761234306.253606:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234306.253609:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000040:0.0:1761234306.253611:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234306.253615:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000001:0.0:1761234306.253617:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000100:00000200:2.0:1761234306.253622:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234331 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000020:00000001:0.0:1761234306.253624:0:468:0:(genops.c:899:class_conn2export()) Process entered 00000020:00000040:0.0:1761234306.253627:0:468:0:(genops.c:911:class_conn2export()) looking for export cookie 0x3e9d1749687b9cdb 00000020:00000001:0.0:1761234306.253631:0:468:0:(lustre_handles.c:133:class_handle2object()) Process entered 00000020:00000040:0.0:1761234306.253636:0:468:0:(lustre_handles.c:151:class_handle2object()) GET export 00000000a7a4e056 refcount=4 00000100:00000001:2.0:1761234306.253636:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234306.253640:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 0 dl 1761234331 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000020:00000001:0.0:1761234306.253641:0:468:0:(lustre_handles.c:158:class_handle2object()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000020:00000001:0.0:1761234306.253650:0:468:0:(genops.c:913:class_conn2export()) Process leaving (rc=18446639356083156992 : -104717626394624 : ffffa0c28713b800) 00000100:00000001:2.0:1761234306.253652:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.253657:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000040:0.0:1761234306.253663:0:468:0:(lustre_net.h:1943:ptlrpc_connection_put()) PUT conn=00000000bbafe9dc refcount 8 to 192.168.201.124@tcp 00000100:00000001:2.0:1761234306.253664:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.253670:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1761234306.253672:0:468:0:(connection.c:140:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1761234306.253673:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.253678:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1761234306.253681:0:468:0:(connection.c:145:ptlrpc_connection_addref()) conn=00000000bbafe9dc refcount 9 to 192.168.201.124@tcp 00000100:00100000:2.0:1761234306.253683:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234306/real 1761234306] req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234306.253688:0:468:0:(connection.c:147:ptlrpc_connection_addref()) Process leaving (rc=18446639356019622656 : -104717689928960 : ffffa0c2834a4300) 00000020:00000040:0.0:1761234306.253696:0:468:0:(genops.c:979:class_export_put()) PUTting export 00000000a7a4e056 : new refcount 3 00000400:00000001:2.0:1761234306.253707:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1761234306.253708:0:468:0:(import.c:615:import_select_connection()) lustre-OST0000-osc-ffffa0c283fe8000: import 00000000256877a5 using connection 192.168.201.124@tcp/192.168.201.124@tcp 00000400:00000001:2.0:1761234306.253711:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1761234306.253716:0:468:0:(import.c:619:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234306.253716:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234306.253720:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 02000000:00000001:0.0:1761234306.253722:0:468:0:(sec.c:1458:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1761234306.253723:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234306.253727:0:88993:0:(events.c:103:reply_in_callback()) Process entered 02000000:00000001:0.0:1761234306.253729:0:468:0:(sec.c:1498:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1761234306.253733:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 02000000:00000001:0.0:1761234306.253738:0:468:0:(sec.c:1526:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1761234306.253743:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:0.0:1761234306.253744:0:468:0:(obd_class.h:857:obd_reconnect()) Process entered 00000008:00100000:0.0:1761234306.253748:0:468:0:(osc_request.c:3485:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34539264 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1761234306.253750:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234306.253751:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1761234306.253758:0:468:0:(osc_request.c:3488:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.253758:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234306.253762:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234306.253764:0:468:0:(obd_class.h:867:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234306.253769:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000010:0.0:1761234306.253771:0:468:0:(client.c:538:ptlrpc_request_cache_alloc()) slab-alloced 'req': 1192 at 00000000eff3cf06. 00000020:00000040:0.0:1761234306.253779:0:468:0:(genops.c:1191:class_import_get()) import 00000000256877a5 refcount=5 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000100:00000001:2.0:1761234306.253779:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1761234306.253786:0:468:0:(sec.c:1867:sptlrpc_get_sepol()) Process entered 02000000:00000001:0.0:1761234306.253789:0:468:0:(sec.c:1879:sptlrpc_get_sepol()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1761234306.253789:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:0.0:1761234306.253799:0:468:0:(client.c:780:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1761234306.253800:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1761234306.253806:0:468:0:(sec.c:437:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1761234306.253806:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.253811:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1761234306.253812:0:468:0:(sec.c:455:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.253813:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.253821:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234306.253823:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:0.0:1761234306.253825:0:468:0:(sec_null.c:155:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000d6261eeb. 00000100:00080000:2.0:1761234306.253827:0:88993:0:(import.c:86:import_set_state_nolock()) 0000000014ae8b98 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1761234306.253833:0:468:0:(client.c:860:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1761234306.253833:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234306.253840:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234306.253874:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234306.253880:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@000000003c218247 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f94b4173-c622-408c-b948-b77c5482236b:88993:1846784245236544:192.168.201.124@tcp:250: 00000100:00100000:0.0:1761234306.253886:0:468:0:(import.c:807:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.253886:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.253891:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234306.253896:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@000000003c218247 x1846784245236544/t0(0) o250->MGC192.168.201.124@tcp@192.168.201.124@tcp:26/25 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:0.0:1761234306.253944:0:468:0:(jobid.c:870:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1761234306.253944:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1761234306.253949:0:468:0:(jobid.c:907:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234306.253950:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234306.253953:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234306.253957:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234306.253959:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000100:00000040:0.0:1761234306.253960:0:468:0:(ptlrpcd.c:301:ptlrpcd_add_req()) @@@ add req [00000000eff3cf06] to pc [ptlrpcd_rcv+-1] req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000020:00000040:2.0:1761234306.253961:0:88993:0:(genops.c:1204:class_import_put()) import 0000000014ae8b98 refcount=3 obd=MGC192.168.201.124@tcp 00000020:00000001:2.0:1761234306.253968:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234306.253970:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 000000005851ba37. 02000000:00000001:2.0:1761234306.253976:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234306.253978:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234306.253979:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 000000003c218247. 00000100:00000001:0.0:1761234306.253983:0:468:0:(import.c:814:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.253984:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234306.253986:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.253989:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1761234306.253991:0:468:0:(recover.c:60:ptlrpc_initiate_recovery()) Process leaving 00000100:00000040:0.0:1761234306.253998:0:468:0:(pinger.c:320:ptlrpc_pinger_main()) next wakeup in 5 (6785) 00000100:00000001:2.0:1761234306.253999:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234306.254001:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.254006:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234306.254008:0:88993:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234306.254009:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234306.254013:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254019:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234306.254020:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234306.254022:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234306.254023:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234306.254026:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@0000000070caacfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236608:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234306.254030:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234306.254032:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234306.254032:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234306.254034:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234306.254036:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234306.254037:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234306.254039:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234306.254041:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234306.254043:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236608, portal 10 00000100:00000001:2.0:1761234306.254044:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234306.254045:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246301824 : -104717463249792 : ffffa0c290cd1c80) 00000100:00000040:2.0:1761234306.254050:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234331 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254054:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234306.254055:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234306.254057:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234306.254058:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1846784245236608, offset 0 00000400:00000200:2.0:1761234306.254061:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234306.254064:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234306.254066:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234306.254068:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234306.254070:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234306.254071:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11914:11914:256:4294967295] 192.168.201.124@tcp LPNI seq info [11914:11914:8:4294967295] 00000400:00000200:2.0:1761234306.254077:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234306.254080:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234306.254082:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234306.254084:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234306.254086:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761234306.254090:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234306.254091:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1761234306.254094:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234306.254095:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234306.254097:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234306.254099:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234306.254101:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234306.254102:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234306.254103:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234306.254105:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234331 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254109:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234306.254110:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 0 dl 1761234331 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254114:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.254115:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234306.254117:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.254118:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234306.254119:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.254120:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234306.254122:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234306/real 1761234306] req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234306.254127:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234306.254128:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234306.254129:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234306.254130:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234306.254130:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234306.254131:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234306.254132:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234306.254137:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254141:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234306.254142:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254142:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234306.254143:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234306.254144:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254149:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234306.254151:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234306.254155:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234306.254156:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254157:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234306.254157:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254158:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234306.254164:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234306.254166:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000ef40b234 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234306.254168:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234306.254169:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234306.254170:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234306.254174:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@0000000070caacfc pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236608:192.168.201.124@tcp:38: 00000100:00000001:2.0:1761234306.254178:0:88993:0:(client.c:1628:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1761234306.254179:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254184:0:88993:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234306.254184:0:88993:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234306.254186:0:88993:0:(sec.c:667:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1761234306.254186:0:88993:0:(sec.c:699:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234306.254188:0:88993:0:(client.c:1733:ptlrpc_send_new_req()) Sending RPC req@00000000eff3cf06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236672:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234306.254189:0:88993:0:(niobuf.c:715:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1761234306.254190:0:88993:0:(sec.c:1004:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1761234306.254191:0:88993:0:(sec.c:1041:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1761234306.254192:0:88993:0:(sec.c:1716:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1761234306.254194:0:88993:0:(sec_null.c:198:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234306.254195:0:88993:0:(sec.c:1726:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1761234306.254197:0:88993:0:(lib-me.c:92:LNetMEAttach()) slab-alloced 'me' at 000000008c129287. 00000400:00000010:2.0:1761234306.254199:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 00000000925d9f1f. 00000100:00000200:2.0:1761234306.254200:0:88993:0:(niobuf.c:922:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1846784245236672, portal 4 00000100:00000001:2.0:1761234306.254202:0:88993:0:(client.c:2989:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1761234306.254202:0:88993:0:(client.c:2991:ptlrpc_request_addref()) Process leaving (rc=18446639356246294528 : -104717463257088 : ffffa0c290cd0000) 00000100:00000040:2.0:1761234306.254220:0:88993:0:(niobuf.c:942:ptl_send_rpc()) @@@ send flags=0 req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234331 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254224:0:88993:0:(niobuf.c:55:ptl_send_buf()) Process entered 00000100:00000040:2.0:1761234306.254225:0:88993:0:(niobuf.c:58:ptl_send_buf()) peer_id 12345-192.168.201.124@tcp 00000400:00000010:2.0:1761234306.254227:0:88993:0:(lib-md.c:185:lnet_md_build()) slab-alloced 'md' of size 136 at 0000000019b314ed. 00000100:00000200:2.0:1761234306.254228:0:88993:0:(niobuf.c:86:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1846784245236672, offset 0 00000400:00000200:2.0:1761234306.254230:0:88993:0:(lib-move.c:5065:LNetPut()) LNetPut -> 12345-192.168.201.124@tcp 00000400:00000200:2.0:1761234306.254233:0:88993:0:(lib-move.c:2899:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.124@tcp local destination 00000400:00000200:2.0:1761234306.254236:0:88993:0:(lib-move.c:1758:lnet_get_best_ni()) selected best_ni 192.168.201.24@tcp 00000400:00000200:2.0:1761234306.254237:0:88993:0:(lib-move.c:1336:lnet_select_peer_ni()) 192.168.201.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1761234306.254239:0:88993:0:(lib-move.c:1416:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.124@tcp 00000400:00000200:2.0:1761234306.254240:0:88993:0:(lib-move.c:1863:lnet_handle_send()) 192.168.201.24@tcp NI seq info: [11915:11915:256:4294967295] 192.168.201.124@tcp LPNI seq info [11915:11915:8:4294967295] 00000400:00000200:2.0:1761234306.254245:0:88993:0:(lib-move.c:1975:lnet_handle_send()) TRACE: 192.168.201.24@tcp(192.168.201.24@tcp:) -> 192.168.201.124@tcp(192.168.201.124@tcp:192.168.201.124@tcp) : PUT try# 0 00000800:00000200:2.0:1761234306.254248:0:88993:0:(socklnd_cb.c:995:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234306.254249:0:88993:0:(socklnd_cb.c:53:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000be9b4ec (tot 1676530). 00000800:00000200:2.0:1761234306.254252:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.0:1761234306.254253:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000200:2.1:1761234306.254256:0:88993:0:(socklnd.c:239:ksocknal_find_peer_locked()) got peer_ni [0000000062b0d59a] -> 12345-192.168.201.124@tcp (2) 00000800:00000200:2.1:1761234306.254257:0:88993:0:(socklnd_cb.c:851:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.124 (cnted 0, interval 60, 42 secs later) 00000800:00000100:2.0:1761234306.254260:0:88993:0:(socklnd_cb.c:970:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.124@tcp 00000800:00000010:2.0:1761234306.254261:0:88993:0:(socklnd_cb.c:110:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000be9b4ec (tot 1676282). 00000400:00000200:2.0:1761234306.254262:0:88993:0:(lib-msg.c:1070:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1761234306.254264:0:88993:0:(lib-msg.c:838:lnet_health_check()) health check: 192.168.201.24@tcp->192.168.201.124@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1761234306.254266:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 0000000019b314ed 00000400:00000010:2.0:1761234306.254267:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 0000000019b314ed. 00000100:00000001:2.0:1761234306.254268:0:88993:0:(events.c:53:request_out_callback()) Process entered 00000100:00000200:2.0:1761234306.254270:0:88993:0:(events.c:64:request_out_callback()) @@@ type 5, status -5 req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234331 ref 2 fl Rpc:Nr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254275:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234306.254276:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 1 req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 0 dl 1761234331 ref 2 fl Rpc:eNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254280:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.254281:0:88993:0:(events.c:93:request_out_callback()) Process leaving 00000100:00000001:2.0:1761234306.254282:0:88993:0:(niobuf.c:103:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.254283:0:88993:0:(niobuf.c:951:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1761234306.254284:0:88993:0:(client.c:1755:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.254285:0:88993:0:(client.c:2281:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1761234306.254286:0:88993:0:(client.c:2295:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1761234306/real 1761234306] req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000400:00000001:2.0:1761234306.254291:0:88993:0:(lib-msg.c:43:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1761234306.254292:0:88993:0:(lib-msg.c:52:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1761234306.254293:0:88993:0:(lib-me.c:135:lnet_me_unlink()) slab-freed 'me' at 000000008c129287. 00000400:00000200:2.0:1761234306.254294:0:88993:0:(lib-md.c:68:lnet_md_unlink()) Unlinking md 00000000925d9f1f 00000400:00000010:2.0:1761234306.254294:0:88993:0:(lib-lnet.h:313:lnet_md_free()) slab-freed 'md' at 00000000925d9f1f. 00000100:00000001:2.0:1761234306.254295:0:88993:0:(events.c:103:reply_in_callback()) Process entered 00000100:00000200:2.0:1761234306.254296:0:88993:0:(events.c:105:reply_in_callback()) @@@ type 6, status 0 req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQr/0/ffffffff rc 0/-1 job:'' 00000100:00000200:2.0:1761234306.254300:0:88993:0:(events.c:127:reply_in_callback()) @@@ unlink req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254304:0:88993:0:(events.c:185:reply_in_callback()) Process leaving 00000100:00000001:2.0:1761234306.254304:0:88993:0:(client.c:2761:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254305:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234306.254306:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1761234306.254308:0:88993:0:(client.c:2326:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:eXNQU/0/ffffffff rc 0/-1 job:'' 00000100:00000001:2.0:1761234306.254311:0:88993:0:(client.c:2331:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1761234306.254313:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Rpc:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234306.254316:0:88993:0:(client.c:1932:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1761234306.254318:0:88993:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254319:0:88993:0:(niobuf.c:445:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1761234306.254319:0:88993:0:(niobuf.c:459:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254321:0:88993:0:(import.c:1008:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1761234306.254321:0:88993:0:(import.c:1038:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1761234306.254323:0:88993:0:(import.c:86:import_set_state_nolock()) 00000000256877a5 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1761234306.254324:0:88993:0:(import.c:1429:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.124@tcp failed (-110) 00000100:00000001:2.0:1761234306.254325:0:88993:0:(import.c:1435:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1761234306.254327:0:88993:0:(lustre_net.h:2402:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Interpret:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00100000:2.0:1761234306.254330:0:88993:0:(client.c:2215:ptlrpc_check_set()) Completed RPC req@00000000eff3cf06 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e8fc2de5-c58c-4d00-8403-987fe90bf686:88993:1846784245236672:192.168.201.124@tcp:8: 00000100:00000001:2.0:1761234306.254332:0:88993:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254333:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234306.254334:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@0000000070caacfc x1846784245236608/t0(0) o38->lustre-MDT0000-mdc-ffffa0c283fe8000@192.168.201.124@tcp:12/10 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234306.254338:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234306.254338:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234306.254339:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000001854c0bd. 02000000:00000001:2.0:1761234306.254340:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234306.254341:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234306.254341:0:88993:0:(genops.c:1204:class_import_put()) import 00000000ef40b234 refcount=204 obd=lustre-MDT0000-mdc-ffffa0c283fe8000 00000020:00000001:2.0:1761234306.254343:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234306.254343:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 0000000078138607. 02000000:00000001:2.0:1761234306.254346:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234306.254346:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234306.254347:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 0000000070caacfc. 00000100:00000001:2.0:1761234306.254349:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234306.254350:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254351:0:88993:0:(client.c:2663:__ptlrpc_req_finished()) Process entered 00000100:00000040:2.0:1761234306.254352:0:88993:0:(client.c:2671:__ptlrpc_req_finished()) @@@ refcount now 0 req@00000000eff3cf06 x1846784245236672/t0(0) o8->lustre-OST0000-osc-ffffa0c283fe8000@192.168.201.124@tcp:28/4 lens 520/544 e 0 to 1 dl 1761234331 ref 1 fl Complete:EeXNQU/0/ffffffff rc -110/-1 job:'' 00000100:00000001:2.0:1761234306.254356:0:88993:0:(client.c:2578:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1761234306.254357:0:88993:0:(sec.c:1738:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1761234306.254358:0:88993:0:(sec_null.c:212:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 000000006ca8bb96. 02000000:00000001:2.0:1761234306.254360:0:88993:0:(sec.c:1752:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1761234306.254361:0:88993:0:(genops.c:1198:class_import_put()) Process entered 00000020:00000040:2.0:1761234306.254361:0:88993:0:(genops.c:1204:class_import_put()) import 00000000256877a5 refcount=4 obd=lustre-OST0000-osc-ffffa0c283fe8000 00000020:00000001:2.0:1761234306.254363:0:88993:0:(genops.c:1211:class_import_put()) Process leaving 02000000:00000010:2.0:1761234306.254363:0:88993:0:(sec_null.c:182:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at 00000000d6261eeb. 02000000:00000001:2.0:1761234306.254365:0:88993:0:(sec.c:469:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1761234306.254365:0:88993:0:(sec.c:486:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1761234306.254366:0:88993:0:(client.c:544:ptlrpc_request_cache_free()) slab-freed '(req)': 1192 at 00000000eff3cf06. 00000100:00000001:2.0:1761234306.254368:0:88993:0:(client.c:2636:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1761234306.254368:0:88993:0:(client.c:2704:__ptlrpc_req_finished()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254369:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.254371:0:88993:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234306.254371:0:88993:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.254372:0:88993:0:(ptlrpcd.c:323:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1761234306.254373:0:88993:0:(ptlrpcd.c:422:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1761234306.765022:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234306.765029:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234306.765032:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234306.765037:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234306.765043:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234306.765047:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.765051:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234306.765053:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234306.765056:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234307.148945:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234307.148953:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234307.148962:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234307.148964:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234307.148967:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234307.148968:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234307.149042:0:90549:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234307.149046:0:90549:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234307.149049:0:90549:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1761234307.788998:0:92344:0:(client.c:2357:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1761234307.789004:0:92344:0:(client.c:2429:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1761234307.789006:0:92344:0:(client.c:2457:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1761234307.789011:0:92344:0:(client.c:2500:ptlrpc_set_wait()) set 00000000cf2314a4 going to sleep for 0 seconds 00000100:00000001:2.0:1761234307.789017:0:92344:0:(client.c:1798:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1761234307.789021:0:92344:0:(client.c:2750:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234307.789024:0:92344:0:(client.c:1239:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1761234307.789027:0:92344:0:(client.c:1296:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1761234307.789029:0:92344:0:(client.c:2266:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234308.173023:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234308.173031:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234308.173038:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234308.173039:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1761234308.173043:0:92377:0:(ldlm_pool.c:478:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1761234308.173044:0:92377:0:(ldlm_pool.c:482:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1761234308.173235:0:90550:0:(genops.c:1831:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1761234308.173240:0:90550:0:(genops.c:1845:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1761234308.173245:0:90550:0:(ldlm_lockd.c:2792:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 55723 lines, 55723 kept, 0 dropped, 0 bad.